canu-1.3/000077500000000000000000000000001272335422300123065ustar00rootroot00000000000000canu-1.3/README.citation000066400000000000000000000003371272335422300150020ustar00rootroot00000000000000Berlin K, Koren S, Chin CS, Drake PJ, Landolin JM, Phillippy AM Assembling Large Genomes with Single-Molecule Sequencing and Locality Sensitive Hashing. Nature Biotechnology. (2015). Stay tuned for a Canu-specific citation canu-1.3/README.license.GPL000066400000000000000000000431311272335422300152320ustar00rootroot00000000000000 GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc. 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. canu-1.3/README.licenses000066400000000000000000000106461272335422300150010ustar00rootroot00000000000000 This software constitutes a joint work and the contributions of individual authors are subject to different licenses. Contributions and licenses are listed in the applicable source files, with specific details on each individual contribution captured in the revision control system. This software is based on RELEASE_1-3_2004-03-17 of the 'Celera Assembler' (http://wgs-assembler.sourceforge.net) as distributed by Applera Corporation under the GNU General Public License, version 2. Canu branched from Celera Assembler at revision 4587. This software is based on the 'kmer package' (http://kmer.sourceforge.net) as distributed by Applera Corporation under the GNU General Public License, version 2. Canu branched from the kmer project at revision 1994. For the full software distribution and modifications made under the GNU General Public License version 2: This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received (README.license.GPL) a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA For modifications made by the Battelle National Biodefense Institute under the BSD 3-Clause License: This Software was prepared for the Department of Homeland Security (DHS) by the Battelle National Biodefense Institute, LLC (BNBI) as part of contract HSHQDC-07-C-00020 to manage and operate the National Biodefense Analysis and Countermeasures Center (NBACC), a Federally Funded Research and Development Center. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the Battelle National Biodefense Institute nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. For modifications in the public domain: PUBLIC DOMAIN NOTICE This software is "United States Government Work" under the terms of the United States Copyright Act. It was written as part of the authors' official duties for the United States Government and thus cannot be copyrighted. This software is freely available to the public for use without a copyright notice. Restrictions cannot be placed on its present or future use. Although all reasonable efforts have been taken to ensure the accuracy and reliability of the software and associated data, the National Human Genome Research Institute (NHGRI), National Institutes of Health (NIH) and the U.S. Government do not and cannot warrant the performance or results that may be obtained by using this software or data. NHGRI, NIH and the U.S. Government disclaim all warranties as to performance, merchantability or fitness for any particular purpose. Please cite the authors in any work or product based on this material. canu-1.3/README.md000066400000000000000000000026471272335422300135760ustar00rootroot00000000000000# Canu Canu is a fork of the [Celera Assembler](http://wgs-assembler.sourceforge.net/wiki/index.php?title=Main_Page), designed for high-noise single-molecule sequencing (such as the [PacBio](http://www.pacb.com) [RS II](http://www.pacb.com/products-and-services/pacbio-systems/rsii/) or [Oxford Nanopore](https://www.nanoporetech.com/) [MinION](https://www.nanoporetech.com/products-services/minion-mki)). Canu is a hierachical assembly pipeline which runs in four steps: * Detect overlaps in high-noise sequences using [MHAP](https://github.com/marbl/MHAP) * Generate corrected sequence consensus * Trim corrected sequences * Assemble trimmed corrected sequences ## Build: git clone https://github.com/marbl/canu.git cd canu/src make -j ## Run: Brief command line help: ..//bin/canu Full list of parameters: ..//bin/canu -options ## Learn: The [quick start](http://canu.readthedocs.io/en/stable/quick-start.html) will get you assembling quickly, while the [tutorial](http://canu.readthedocs.io/en/stable/tutorial.html) explains things in more detail. ## Citation: - Berlin K, Koren S, Chin CS, Drake PJ, Landolin JM, Phillippy AM [Assembling Large Genomes with Single-Molecule Sequencing and Locality Sensitive Hashing](http://www.nature.com/nbt/journal/v33/n6/abs/nbt.3238.html). Nature Biotechnology. (2015). - Stay tuned for a Canu-specific citation canu-1.3/addCopyrights-BuildData.pl000066400000000000000000000056251272335422300173060ustar00rootroot00000000000000#!/usr/local/bin/perl use strict; my @dateStrings = ( "???", "JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC" ); my %dateStrings; $dateStrings{"Jan"} = "01"; $dateStrings{"Feb"} = "02"; $dateStrings{"Mar"} = "03"; $dateStrings{"Apr"} = "04"; $dateStrings{"May"} = "05"; $dateStrings{"Jun"} = "06"; $dateStrings{"Jul"} = "07"; $dateStrings{"Aug"} = "08"; $dateStrings{"Sep"} = "09"; $dateStrings{"Oct"} = "10"; $dateStrings{"Nov"} = "11"; $dateStrings{"Dec"} = "12"; if (! -e "logs") { system("git log --name-status > logs"); } # Update this after each copyright update commit, please. Best method is to commit # the copyright changes -- none of the addCopyrights files, -- update this file and # Then commit the addCopyrights files. my %stoppingCommits; $stoppingCommits{"6950cb74e302a97673a5ba482b3b8992eea72c37"} = 1; # 20 AUG 2015 - Initial copyright addition. $stoppingCommits{"72c27c95d61cb8f37e859c4039456eb2acc5c55b"} = 1; # 19 NOV 2015 - Second copyright addition. $stoppingCommits{"b2df5790f77d38cc31fe77a7f65360e02389f92e"} = 1; # 04 MAR 2016 $stoppingCommits{"1ef335952342ef06ad1651a888f09c312f54dab8"} = 1; # 18 MAY 2016 open(F, "< logs") or die "Failed to open 'logs': $!\n"; $_ = ; chomp; my $author; my $date; while (!eof(F)) { my $commit; if (m/^commit\s+(\w+)$/) { $commit = $1; } else { die "Expected commit line, got '$_'\n"; } last if (exists($stoppingCommits{$commit})); $_ = ; chomp; if (m/^Merge/) { # Merge commits include an extra line here: "Merge: a75ed40 717c0b1" # No files change, and we can just process the rest of the entry as normal. $_ = ; chomp; } if (m/walenz/i) { $author = "Brian P. Walenz"; } elsif (m/koren/i) { $author = "Sergey Koren"; } else { print STDERR "Skipping commit from '$_'\n"; $author = undef; } $_ = ; chomp; if (m/Date:\s+\w+\s+(\w+)\s+(\d+)\s+\d+:\d+:\d+\s+(\d+)/) { my $day = substr("00$2", -2); my $mo = $dateStrings{$1}; my $year = $3; die "Invalid month '$3'\n" if (! defined($mo)); $date = "$year$mo$day" } else { die "Failed to match date in '$date'\n"; } #print STDERR "$commit -- $date -- $author\n"; $_ = ; chomp; while (! m/^commit/) { next if (m/^$/); # Blank line next if (m/^\s+/); # Comment line if ($_ =~ m/M\s+(\S+)$/) { print "A $1 nihh$date$author\n" if (defined($author)); } elsif ($_ =~ m/A\s+(\S+)$/) { # New file, treat as normal. print "A $1 nihh$date$author\n" if (defined($author)); } elsif ($_ =~ m/D\s+(\S+)$/) { # Deleted file, do nothing,. } else { print STDERR "$_\n"; } } continue { $_ = ; chomp; } } close(F); canu-1.3/addCopyrights.dat000066400000000000000000023646121272335422300156220ustar00rootroot00000000000000A kmer/libutil/bzipBuffer.H jcvi20140411Brian P. Walenz A kmer/libutil/bzipBuffer.H jcvi20060623Brian P. Walenz A kmer/libutil/unaryEncoding.h jcvi20140411Brian P. Walenz A kmer/libutil/unaryEncoding.h none20040427Brian P. Walenz A kmer/libutil/palloc.c jcvi20110110Brian P. Walenz A kmer/libutil/palloc.c jcvi20080130Brian P. Walenz A kmer/libutil/palloc.c jcvi20071029Brian P. Walenz A kmer/libutil/palloc.c jcvi20060228Brian P. Walenz A kmer/libutil/palloc.c jcvi20050714Brian P. Walenz A kmer/libutil/palloc.c jcvi20050523Brian P. Walenz A kmer/libutil/palloc.c jcvi20050412Brian P. Walenz A kmer/libutil/palloc.c jcvi20050305Brian P. Walenz A kmer/libutil/palloc.c none20041010Brian P. Walenz A kmer/libutil/palloc.c none20040511Brian P. Walenz A kmer/libutil/palloc.c craa20040506Brian P. Walenz A kmer/libutil/palloc.c craa20040506Brian P. Walenz A kmer/libutil/palloc.c none20040430Brian P. Walenz A kmer/libutil/palloc.c none20040421Brian P. Walenz A kmer/libutil/palloc.c none20040113Brian P. Walenz A kmer/libutil/palloc.c craa20030506Brian P. Walenz A kmer/libutil/palloc.c craa20030102Brian P. Walenz A kmer/libutil/recordFile.C jcvi20140411Brian P. Walenz A kmer/libutil/recordFile.C jcvi20080826Brian P. Walenz A kmer/libutil/recordFile.C jcvi20080821Brian P. Walenz A kmer/libutil/recordFile.C jcvi20080820Brian P. Walenz A kmer/libutil/recordFile.C jcvi20080817Brian P. Walenz A kmer/libutil/recordFile.C jcvi20080814Brian P. Walenz A kmer/libutil/recordFile.C jcvi20080805Brian P. Walenz A kmer/libutil/recordFile.C jcvi20080724Brian P. Walenz A kmer/libutil/recordFile.C jcvi20080724Brian P. Walenz A kmer/libutil/recordFile.C jcvi20080708Brian P. Walenz A kmer/libutil/fibonacciEncoding.h jcvi20140411Brian P. Walenz A kmer/libutil/fibonacciEncoding.h jcvi20080609Brian P. Walenz A kmer/libutil/fibonacciEncoding.h none20040427Brian P. Walenz A kmer/libutil/util.c jcvi20140411Brian P. Walenz A kmer/libutil/util.c jcvi20100919Brian P. Walenz A kmer/libutil/util.c jcvi20080912Brian P. Walenz A kmer/libutil/util++.H bnbi20141205Brian P. Walenz A kmer/libutil/util++.H jcvi20140411Brian P. Walenz A kmer/libutil/util++.H jcvi20080708Brian P. Walenz A kmer/libutil/util++.H jcvi20060706Brian P. Walenz A kmer/libutil/util++.H jcvi20060624Brian P. Walenz A kmer/libutil/util++.H jcvi20060613Brian P. Walenz A kmer/libutil/util++.H jcvi20060425Brian P. Walenz A kmer/libutil/util++.H jcvi20051127Brian P. Walenz A kmer/libutil/util++.H jcvi20050712Brian P. Walenz A kmer/libutil/util++.H jcvi20050506Brian P. Walenz A kmer/libutil/util++.H jcvi20050401Brian P. Walenz A kmer/libutil/util++.H jcvi20050314Brian P. Walenz A kmer/libutil/util++.H jcvi20050310Brian P. Walenz A kmer/libutil/util++.H jcvi20050309Brian P. Walenz A kmer/libutil/util++.H jcvi20050209Brian P. Walenz A kmer/libutil/util++.H jcvi20050207Brian P. Walenz A kmer/libutil/util++.H none20041010Brian P. Walenz A kmer/libutil/util++.H none20040818Brian P. Walenz A kmer/libutil/util++.H craa20040802Brian P. Walenz A kmer/libutil/util++.H none20040610Brian P. Walenz A kmer/libutil/util++.H none20040525Brian P. Walenz A kmer/libutil/util++.H none20040524Brian P. Walenz A kmer/libutil/util++.H none20040514Brian P. Walenz A kmer/libutil/util++.H none20040511Brian P. Walenz A kmer/libutil/util++.H none20040511Brian P. Walenz A kmer/libutil/util++.H craa20040506Brian P. Walenz A kmer/libutil/util++.H none20040430Brian P. Walenz A kmer/libutil/util++.H none20040427Brian P. Walenz A kmer/libutil/util++.H none20040421Brian P. Walenz A kmer/libutil/logMsg.H jcvi20140411Brian P. Walenz A kmer/libutil/logMsg.H jcvi20130329Brian P. Walenz A kmer/libutil/logMsg.H jcvi20120510Brian P. Walenz A kmer/libutil/logMsg.H jcvi20120508Brian P. Walenz A kmer/libutil/logMsg.H jcvi20070102Brian P. Walenz A kmer/libutil/logMsg.H jcvi20061108Brian P. Walenz A kmer/libutil/logMsg.H jcvi20060706Brian P. Walenz A kmer/libutil/logMsg.H jcvi20060706Brian P. Walenz A kmer/libutil/logMsg.H jcvi20050712Brian P. Walenz A kmer/libutil/logMsg.H jcvi20050305Brian P. Walenz A kmer/libutil/logMsg.H none20041010Brian P. Walenz A kmer/libutil/logMsg.H none20040614Brian P. Walenz A kmer/libutil/logMsg.H none20040610Brian P. Walenz A kmer/libutil/endianess.H jcvi20140411Brian P. Walenz A kmer/libutil/endianess.H jcvi20071226Brian P. Walenz A kmer/libutil/endianess.H jcvi20060724Brian P. Walenz A kmer/libutil/endianess.H jcvi20060512Brian P. Walenz A kmer/libutil/endianess.H jcvi20051127Brian P. Walenz A kmer/libutil/eliasGammaEncoding.h jcvi20140411Brian P. Walenz A kmer/libutil/eliasGammaEncoding.h none20040427Brian P. Walenz A kmer/libutil/fibonacciNumbers.C jcvi20140411Brian P. Walenz A kmer/libutil/fibonacciNumbers.C none20041010Brian P. Walenz A kmer/libutil/fibonacciNumbers.C none20040427Brian P. Walenz A kmer/libutil/uint32List.H jcvi20140411Brian P. Walenz A kmer/libutil/uint32List.H jcvi20060613Brian P. Walenz D kmer/libutil/uint32List.H kmer/libutil/u32bitList.H A kmer/libutil/qsort_mt.c jcvi20080724Brian P. Walenz A kmer/libutil/qsort_mt.c jcvi20080609Brian P. Walenz A kmer/libutil/util.h jcvi20140411Brian P. Walenz A kmer/libutil/util.h jcvi20120118Brian P. Walenz A kmer/libutil/util.h jcvi20111229Brian P. Walenz A kmer/libutil/util.h jcvi20110110Brian P. Walenz A kmer/libutil/util.h jcvi20100919Brian P. Walenz A kmer/libutil/util.h jcvi20080912Brian P. Walenz A kmer/libutil/util.h jcvi20080828Brian P. Walenz A kmer/libutil/util.h jcvi20080817Brian P. Walenz A kmer/libutil/util.h jcvi20080814Brian P. Walenz A kmer/libutil/util.h jcvi20080609Brian P. Walenz A kmer/libutil/util.h jcvi20080514Brian P. Walenz A kmer/libutil/util.h jcvi20080130Brian P. Walenz A kmer/libutil/util.h jcvi20071029Brian P. Walenz A kmer/libutil/util.h jcvi20070823Brian P. Walenz A kmer/libutil/util.h jcvi20070320Brian P. Walenz A kmer/libutil/util.h jcvi20060623Brian P. Walenz A kmer/libutil/util.h jcvi20060621Brian P. Walenz A kmer/libutil/util.h jcvi20060308Brian P. Walenz A kmer/libutil/util.h jcvi20060228Brian P. Walenz A kmer/libutil/util.h jcvi20051127Brian P. Walenz A kmer/libutil/util.h jcvi20050808Brian P. Walenz A kmer/libutil/util.h jcvi20050712Brian P. Walenz A kmer/libutil/util.h jcvi20050712Brian P. Walenz A kmer/libutil/util.h jcvi20050523Brian P. Walenz A kmer/libutil/util.h jcvi20050506Brian P. Walenz A kmer/libutil/util.h jcvi20050412Brian P. Walenz A kmer/libutil/util.h jcvi20050320Brian P. Walenz A kmer/libutil/util.h jcvi20050313Brian P. Walenz A kmer/libutil/util.h jcvi20050310Brian P. Walenz A kmer/libutil/util.h jcvi20050309Brian P. Walenz A kmer/libutil/util.h jcvi20050207Brian P. Walenz A kmer/libutil/util.h none20041010Brian P. Walenz A kmer/libutil/util.h none20040818Brian P. Walenz A kmer/libutil/util.h craa20040802Brian P. Walenz A kmer/libutil/util.h none20040528Brian P. Walenz A kmer/libutil/util.h none20040511Brian P. Walenz A kmer/libutil/util.h craa20040506Brian P. Walenz A kmer/libutil/util.h craa20040506Brian P. Walenz A kmer/libutil/util.h none20040421Brian P. Walenz A kmer/libutil/util.h none20040421Brian P. Walenz A kmer/libutil/recordFile.H jcvi20140411Brian P. Walenz A kmer/libutil/recordFile.H jcvi20080820Brian P. Walenz A kmer/libutil/recordFile.H jcvi20080818Brian P. Walenz A kmer/libutil/recordFile.H jcvi20080817Brian P. Walenz A kmer/libutil/recordFile.H jcvi20080814Brian P. Walenz A kmer/libutil/recordFile.H jcvi20080805Brian P. Walenz A kmer/libutil/recordFile.H jcvi20080708Brian P. Walenz A kmer/libutil/unaryEncodingTester.C jcvi20140411Brian P. Walenz A kmer/libutil/unaryEncodingTester.C jcvi20080129Brian P. Walenz A kmer/libutil/unaryEncodingTester.C none20041010Brian P. Walenz A kmer/libutil/unaryEncodingTester.C none20040427Brian P. Walenz A kmer/libutil/bzipBuffer.C jcvi20140411Brian P. Walenz A kmer/libutil/bzipBuffer.C jcvi20060623Brian P. Walenz A kmer/libutil/file.c jcvi20140411Brian P. Walenz A kmer/libutil/file.c jcvi20121015Brian P. Walenz A kmer/libutil/file.c jcvi20111229Brian P. Walenz A kmer/libutil/file.c jcvi20080318Brian P. Walenz A kmer/libutil/file.c jcvi20070320Brian P. Walenz A kmer/libutil/file.c jcvi20060308Brian P. Walenz A kmer/libutil/file.c jcvi20060228Brian P. Walenz A kmer/libutil/file.c jcvi20050712Brian P. Walenz A kmer/libutil/file.c jcvi20050309Brian P. Walenz A kmer/libutil/file.c none20041010Brian P. Walenz A kmer/libutil/file.c none20040813Brian P. Walenz A kmer/libutil/file.c none20040812Brian P. Walenz A kmer/libutil/file.c craa20040802Brian P. Walenz A kmer/libutil/file.c none20040430Brian P. Walenz A kmer/libutil/file.c none20040421Brian P. Walenz A kmer/libutil/file.c craa20040401Brian P. Walenz A kmer/libutil/file.c none20040330Brian P. Walenz A kmer/libutil/file.c none20040330Brian P. Walenz A kmer/libutil/file.c craa20040218Clark Mobarry A kmer/libutil/file.c none20040120Brian P. Walenz A kmer/libutil/file.c none20040112Brian P. Walenz A kmer/libutil/file.c craa20040106Brian P. Walenz A kmer/libutil/file.c craa20030908Brian P. Walenz A kmer/libutil/file.c craa20030721Brian P. Walenz A kmer/libutil/file.c craa20030624Brian P. Walenz A kmer/libutil/file.c craa20030507Brian P. Walenz A kmer/libutil/file.c craa20030506Brian P. Walenz A kmer/libutil/eliasDeltaEncoding.h jcvi20140411Brian P. Walenz A kmer/libutil/eliasDeltaEncoding.h none20040427Brian P. Walenz A kmer/libutil/test/test-palloc.c none20041010Brian P. Walenz A kmer/libutil/test/test-palloc.c craa20040506Brian P. Walenz A kmer/libutil/test/test-palloc.c craa20040506Brian P. Walenz A kmer/libutil/test/test-intervalList.C bnbi20141014Brian P. Walenz A kmer/libutil/test/test-intervalList.C jcvi20140411Brian P. Walenz A kmer/libutil/test/test-intervalList.C jcvi20061027Brian P. Walenz A kmer/libutil/test/test-intervalList.C jcvi20061023Brian P. Walenz A kmer/libutil/test/test-intervalList.C jcvi20061023Brian P. Walenz A kmer/libutil/test/test-intervalList.C none20041010Brian P. Walenz A kmer/libutil/test/test-intervalList.C craa20040506Brian P. Walenz A kmer/libutil/test/test-recordFile.C jcvi20140411Brian P. Walenz A kmer/libutil/test/test-recordFile.C jcvi20080909Brian P. Walenz A kmer/libutil/test/test-recordFile.C jcvi20080724Brian P. Walenz A kmer/libutil/test/endianess.c jcvi20140411Brian P. Walenz A kmer/libutil/test/endianess.c jcvi20060724Brian P. Walenz A kmer/libutil/test/atomic.C jcvi20101102Brian P. Walenz A kmer/libutil/test/atomic.C jcvi20060724Brian P. Walenz A kmer/libutil/test/test-bitPackedFile.C jcvi20140411Brian P. Walenz A kmer/libutil/test/test-bitPackedFile.C jcvi20070320Brian P. Walenz A kmer/libutil/test/test-bitPackedFile.C jcvi20061022Brian P. Walenz A kmer/libutil/test/test-bitPackedFile.C none20041010Brian P. Walenz A kmer/libutil/test/test-bitPackedFile.C craa20040506Brian P. Walenz A kmer/libutil/test/test-bitPackedFile.C craa20040506Brian P. Walenz A kmer/libutil/test/test-mmap.c jcvi20140411Brian P. Walenz A kmer/libutil/test/test-mmap.c none20041010Brian P. Walenz A kmer/libutil/test/test-mmap.c craa20040506Brian P. Walenz A kmer/libutil/test/order.C jcvi20140411Brian P. Walenz A kmer/libutil/test/order.C jcvi20051204Brian P. Walenz A kmer/libutil/test/test-bitPacking.C jcvi20140411Brian P. Walenz A kmer/libutil/test/test-bitPacking.C jcvi20071205Brian P. Walenz A kmer/libutil/test/test-bitPacking.C none20041010Brian P. Walenz A kmer/libutil/test/test-bitPacking.C none20040524Brian P. Walenz A kmer/libutil/test/test-bitPacking.C craa20040506Brian P. Walenz A kmer/libutil/test/test-bitPacking.C craa20040506Brian P. Walenz A kmer/libutil/test/tcat.C jcvi20080909Brian P. Walenz A kmer/libutil/test/tcat.C jcvi20070102Brian P. Walenz A kmer/libutil/test/tcat.C jcvi20060308Brian P. Walenz A kmer/libutil/test/tcat.C jcvi20060303Brian P. Walenz A kmer/libutil/test/tcat.C jcvi20060302Brian P. Walenz A kmer/libutil/test/test-bigQueue.C jcvi20050309Brian P. Walenz A kmer/libutil/test/test-bitPackedArray.C jcvi20140411Brian P. Walenz A kmer/libutil/test/test-bitPackedArray.C jcvi20070320Brian P. Walenz A kmer/libutil/test/test-bitPackedArray.C jcvi20061026Brian P. Walenz A kmer/libutil/test/test-bitPackedArray.C jcvi20061022Brian P. Walenz A kmer/libutil/test/test-bitPackedArray.C jcvi20050207Brian P. Walenz A kmer/libutil/test/test-types.c jcvi20140411Brian P. Walenz A kmer/libutil/test/test-types.c none20041010Brian P. Walenz A kmer/libutil/test/test-types.c craa20040506Brian P. Walenz A kmer/libutil/test/test-types.c craa20040506Brian P. Walenz A kmer/libutil/test/test-bzipBuffer.C jcvi20140411Brian P. Walenz A kmer/libutil/test/test-bzipBuffer.C jcvi20060623Brian P. Walenz A kmer/libutil/test/test-freeDiskSpace.c none20041010Brian P. Walenz A kmer/libutil/test/test-freeDiskSpace.c craa20040802Brian P. Walenz A kmer/libutil/test/test-readBuffer.C jcvi20140411Brian P. Walenz A kmer/libutil/test/test-readBuffer.C jcvi20080912Brian P. Walenz A kmer/libutil/test/test-readBuffer.C jcvi20080911Brian P. Walenz A kmer/libutil/test/test-readBuffer.C jcvi20080909Brian P. Walenz A kmer/libutil/test/test-readBuffer.C none20041010Brian P. Walenz A kmer/libutil/test/test-readBuffer.C craa20040506Brian P. Walenz A kmer/libutil/test/test-logMsg.C jcvi20060706Brian P. Walenz A kmer/libutil/test/test-logMsg.C none20040610Brian P. Walenz A kmer/libutil/generalizedUnaryEncoding.h jcvi20140411Brian P. Walenz A kmer/libutil/generalizedUnaryEncoding.h jcvi20070102Brian P. Walenz A kmer/libutil/generalizedUnaryEncoding.h none20040427Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20140411Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20081016Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20080828Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20080808Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20080303Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20080214Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20071217Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20071211Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20071111Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20061211Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20060710Brian P. Walenz A kmer/libkmer/positionDB-access.C jcvi20050207Brian P. Walenz A kmer/libkmer/positionDB-access.C none20041010Brian P. Walenz A kmer/libkmer/positionDB-access.C none20040715Brian P. Walenz A kmer/libkmer/positionDB-access.C none20040511Brian P. Walenz A kmer/libkmer/positionDB-access.C none20040421Brian P. Walenz A kmer/libkmer/positionDB-access.C craa20030814Brian P. Walenz A kmer/libkmer/positionDB-access.C craa20030506Brian P. Walenz A kmer/libkmer/positionDB-access.C craa20030102Brian P. Walenz A kmer/libkmer/positionDB-dump.C jcvi20140411Brian P. Walenz A kmer/libkmer/positionDB-dump.C jcvi20081016Brian P. Walenz A kmer/libkmer/positionDB-dump.C jcvi20080828Brian P. Walenz A kmer/libkmer/positionDB-dump.C jcvi20071211Brian P. Walenz A kmer/libkmer/positionDB-dump.C none20041010Brian P. Walenz A kmer/libkmer/positionDB-dump.C none20040421Brian P. Walenz A kmer/libkmer/positionDB-dump.C craa20030814Brian P. Walenz A kmer/libkmer/positionDB-dump.C craa20030506Brian P. Walenz A kmer/libkmer/positionDB-dump.C craa20030102Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20140411Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080828Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080818Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080814Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080814Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080718Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080707Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080627Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080625Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080623Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080303Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080214Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20080130Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20071211Brian P. Walenz A kmer/libkmer/positionDB-mismatch.C jcvi20071111Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20140411Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20130502Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20130103Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20130103Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20120510Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20110902Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20100828Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20071010Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20070607Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20070326Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20051204Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20051127Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20051127Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20050523Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C jcvi20050320Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C none20040412Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C craa20031020Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C craa20030915Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C craa20030908Brian P. Walenz A kmer/libkmer/existDB-create-from-meryl.C craa20030908Brian P. Walenz A kmer/libkmer/existDB.H jcvi20140411Brian P. Walenz A kmer/libkmer/existDB.H jcvi20130502Brian P. Walenz A kmer/libkmer/existDB.H jcvi20130329Brian P. Walenz A kmer/libkmer/existDB.H jcvi20130308Brian P. Walenz A kmer/libkmer/existDB.H jcvi20130103Brian P. Walenz A kmer/libkmer/existDB.H jcvi20120510Brian P. Walenz A kmer/libkmer/existDB.H jcvi20120508Brian P. Walenz A kmer/libkmer/existDB.H none20101206Liliana Florea A kmer/libkmer/existDB.H jcvi20071029Brian P. Walenz A kmer/libkmer/existDB.H jcvi20071010Brian P. Walenz A kmer/libkmer/existDB.H jcvi20070607Brian P. Walenz A kmer/libkmer/existDB.H jcvi20070326Brian P. Walenz A kmer/libkmer/existDB.H jcvi20051204Brian P. Walenz A kmer/libkmer/existDB.H jcvi20051127Brian P. Walenz A kmer/libkmer/existDB.H jcvi20051127Brian P. Walenz A kmer/libkmer/existDB.H jcvi20050329Brian P. Walenz A kmer/libkmer/existDB.H jcvi20050320Brian P. Walenz A kmer/libkmer/existDB.H none20041010Brian P. Walenz A kmer/libkmer/existDB.H none20040511Brian P. Walenz A kmer/libkmer/existDB.H none20040421Brian P. Walenz A kmer/libkmer/existDB.H craa20031020Brian P. Walenz A kmer/libkmer/existDB.H craa20030908Brian P. Walenz A kmer/libkmer/existDB.H craa20030805Brian P. Walenz A kmer/libkmer/existDB.H craa20030220Brian P. Walenz A kmer/libkmer/existDB.H craa20030213Brian P. Walenz A kmer/libkmer/existDB.H craa20030102Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20140411Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20110330Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20080925Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20080912Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20080909Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20080902Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20080814Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20080814Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20080303Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20080214Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20080130Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20071208Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20071028Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20071010Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20070913Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20070823Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20070817Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20070326Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20051204Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20050913Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20050911Brian P. Walenz A kmer/libkmer/driver-posDB.C jcvi20050519Brian P. Walenz A kmer/libkmer/driver-posDB.C none20041010Brian P. Walenz A kmer/libkmer/driver-posDB.C none20040524Brian P. Walenz A kmer/libkmer/driver-posDB.C none20040514Brian P. Walenz A kmer/libkmer/driver-posDB.C none20040511Brian P. Walenz A kmer/libkmer/driver-posDB.C none20040430Brian P. Walenz A kmer/libkmer/driver-posDB.C none20040430Brian P. Walenz A kmer/libkmer/driver-posDB.C craa20030918Brian P. Walenz A kmer/libkmer/driver-posDB.C craa20030915Brian P. Walenz A kmer/libkmer/driver-posDB.C craa20030814Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20140411Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20081016Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080909Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080902Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080828Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080814Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080814Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080808Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080627Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080625Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080623Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080303Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20080214Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20071217Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20071211Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20071208Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20071111Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20071010Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20070326Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20061211Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20060707Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20051204Brian P. Walenz A kmer/libkmer/positionDB.H jcvi20050913Brian P. Walenz A kmer/libkmer/positionDB.H none20041010Brian P. Walenz A kmer/libkmer/positionDB.H none20040715Brian P. Walenz A kmer/libkmer/positionDB.H none20040511Brian P. Walenz A kmer/libkmer/positionDB.H none20040511Brian P. Walenz A kmer/libkmer/positionDB.H none20040421Brian P. Walenz A kmer/libkmer/positionDB.H craa20031021Brian P. Walenz A kmer/libkmer/positionDB.H craa20030814Brian P. Walenz A kmer/libkmer/positionDB.H craa20030506Brian P. Walenz A kmer/libkmer/positionDB.H craa20030415Brian P. Walenz A kmer/libkmer/positionDB.H craa20030402Brian P. Walenz A kmer/libkmer/positionDB.H craa20030102Brian P. Walenz A kmer/libkmer/existDB-state.C jcvi20140411Brian P. Walenz A kmer/libkmer/existDB-state.C jcvi20130103Brian P. Walenz A kmer/libkmer/existDB-state.C jcvi20070607Brian P. Walenz A kmer/libkmer/existDB-state.C jcvi20070326Brian P. Walenz A kmer/libkmer/existDB-state.C jcvi20050320Brian P. Walenz A kmer/libkmer/existDB-state.C none20041010Brian P. Walenz A kmer/libkmer/existDB-state.C none20040421Brian P. Walenz A kmer/libkmer/existDB-state.C none20040412Brian P. Walenz A kmer/libkmer/existDB-state.C craa20030908Brian P. Walenz A kmer/libkmer/existDB-state.C craa20030908Brian P. Walenz A kmer/libkmer/existDB-create-from-sequence.C bnbi20140831Brian P. Walenz A kmer/libkmer/existDB-create-from-sequence.C jcvi20140411Brian P. Walenz A kmer/libkmer/existDB-create-from-sequence.C jcvi20130103Brian P. Walenz A kmer/libkmer/existDB-create-from-sequence.C jcvi20130103Brian P. Walenz A kmer/libkmer/existDB-create-from-sequence.C jcvi20120510Brian P. Walenz A kmer/libkmer/existDB-create-from-sequence.C jcvi20120508Brian P. Walenz A kmer/libkmer/driver-existDB.C jcvi20140411Brian P. Walenz A kmer/libkmer/driver-existDB.C jcvi20130103Brian P. Walenz A kmer/libkmer/driver-existDB.C jcvi20111229Brian P. Walenz A kmer/libkmer/driver-existDB.C jcvi20080909Brian P. Walenz A kmer/libkmer/driver-existDB.C jcvi20071029Brian P. Walenz A kmer/libkmer/driver-existDB.C jcvi20070913Brian P. Walenz A kmer/libkmer/driver-existDB.C jcvi20070823Brian P. Walenz A kmer/libkmer/driver-existDB.C jcvi20070817Brian P. Walenz A kmer/libkmer/driver-existDB.C jcvi20070326Brian P. Walenz A kmer/libkmer/driver-existDB.C jcvi20051204Brian P. Walenz A kmer/libkmer/driver-existDB.C none20041010Brian P. Walenz A kmer/libkmer/driver-existDB.C none20040524Brian P. Walenz A kmer/libkmer/driver-existDB.C none20040511Brian P. Walenz A kmer/libkmer/driver-existDB.C none20040430Brian P. Walenz A kmer/libkmer/driver-existDB.C craa20031020Brian P. Walenz A kmer/libkmer/driver-existDB.C craa20031016Brian P. Walenz A kmer/libkmer/driver-existDB.C craa20030908Brian P. Walenz A kmer/libkmer/driver-existDB.C craa20030805Brian P. Walenz A kmer/libkmer/driver-existDB.C craa20030220Brian P. Walenz A kmer/libkmer/positionDB-sort.C jcvi20140411Brian P. Walenz A kmer/libkmer/positionDB-sort.C jcvi20111229Brian P. Walenz A kmer/libkmer/positionDB-sort.C jcvi20080312Brian P. Walenz A kmer/libkmer/positionDB-sort.C jcvi20080129Brian P. Walenz A kmer/libkmer/positionDB-sort.C jcvi20071217Brian P. Walenz A kmer/libkmer/positionDB-sort.C jcvi20071211Brian P. Walenz A kmer/libkmer/positionDB-sort.C jcvi20071208Brian P. Walenz A kmer/libkmer/positionDB-sort.C jcvi20060707Brian P. Walenz A kmer/libkmer/positionDB-sort.C none20041010Brian P. Walenz A kmer/libkmer/positionDB-sort.C none20040421Brian P. Walenz A kmer/libkmer/positionDB-sort.C craa20030506Brian P. Walenz A kmer/libkmer/positionDB-sort.C craa20030102Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20140411Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20080828Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20080818Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20080817Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20080814Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20071211Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20051204Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20050712Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20050519Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20050427Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20050316Brian P. Walenz A kmer/libkmer/positionDB-file.C jcvi20050312Brian P. Walenz A kmer/libkmer/positionDB-file.C none20040430Brian P. Walenz A kmer/libkmer/positionDB-file.C none20040430Brian P. Walenz A kmer/libkmer/positionDB-file.C craa20040401Brian P. Walenz A kmer/libkmer/positionDB-file.C craa20030814Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20140411Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20121015Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20081001Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080909Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080902Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080828Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080818Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080814Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080814Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080707Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080625Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080318Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080312Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080303Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080214Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20080129Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20071217Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20071211Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20071208Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20071208Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20071111Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20071028Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20071016Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20071010Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20071009Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20070326Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20070320Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20060914Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20060707Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20060706Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20060425Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20060407Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20051206Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20051204Brian P. Walenz A kmer/libkmer/positionDB.C jcvi20050519Brian P. Walenz A kmer/libkmer/positionDB.C none20041010Brian P. Walenz A kmer/libkmer/positionDB.C none20040715Brian P. Walenz A kmer/libkmer/positionDB.C none20040623Brian P. Walenz A kmer/libkmer/positionDB.C none20040524Brian P. Walenz A kmer/libkmer/positionDB.C none20040514Brian P. Walenz A kmer/libkmer/positionDB.C none20040511Brian P. Walenz A kmer/libkmer/positionDB.C none20040430Brian P. Walenz A kmer/libkmer/positionDB.C none20040430Brian P. Walenz A kmer/libkmer/positionDB.C none20040421Brian P. Walenz A kmer/libkmer/positionDB.C craa20040405Brian P. Walenz A kmer/libkmer/positionDB.C craa20040212Clark Mobarry A kmer/libkmer/positionDB.C none20040121Brian P. Walenz A kmer/libkmer/positionDB.C craa20031021Brian P. Walenz A kmer/libkmer/positionDB.C craa20031021Brian P. Walenz A kmer/libkmer/positionDB.C craa20030915Brian P. Walenz A kmer/libkmer/positionDB.C craa20030814Brian P. Walenz A kmer/libkmer/positionDB.C craa20030507Brian P. Walenz A kmer/libkmer/positionDB.C craa20030506Brian P. Walenz A kmer/libkmer/positionDB.C craa20030506Brian P. Walenz A kmer/libkmer/positionDB.C craa20030418Brian P. Walenz A kmer/libkmer/positionDB.C craa20030416Brian P. Walenz A kmer/libkmer/positionDB.C craa20030415Brian P. Walenz A kmer/libkmer/positionDB.C craa20030402Brian P. Walenz A kmer/libkmer/positionDB.C craa20030102Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C bnbi20140831Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20140411Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20130103Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20130103Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20120510Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20080912Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20080909Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20071010Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20070913Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20070817Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20070607Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20070326Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20051204Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20051127Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20051127Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C jcvi20050320Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C none20041010Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C none20040524Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C none20040421Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C craa20031020Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C craa20030908Brian P. Walenz A kmer/libkmer/existDB-create-from-fasta.C craa20030908Brian P. Walenz A kmer/libkmer/percentCovered.C bnbi20141007Brian P. Walenz A kmer/libkmer/percentCovered.C jcvi20140411Brian P. Walenz A kmer/libkmer/percentCovered.C jcvi20080911Brian P. Walenz A kmer/libkmer/percentCovered.C jcvi20080909Brian P. Walenz A kmer/libkmer/percentCovered.C jcvi20071029Brian P. Walenz A kmer/libkmer/percentCovered.C jcvi20070913Brian P. Walenz A kmer/libkmer/percentCovered.C jcvi20070326Brian P. Walenz A kmer/libkmer/percentCovered.C jcvi20070320Brian P. Walenz A kmer/libkmer/percentCovered.C jcvi20060509Brian P. Walenz A kmer/libkmer/existDB.C jcvi20140411Brian P. Walenz A kmer/libkmer/existDB.C jcvi20130502Brian P. Walenz A kmer/libkmer/existDB.C jcvi20130329Brian P. Walenz A kmer/libkmer/existDB.C jcvi20130103Brian P. Walenz A kmer/libkmer/existDB.C jcvi20120510Brian P. Walenz A kmer/libkmer/existDB.C jcvi20120508Brian P. Walenz A kmer/libkmer/existDB.C jcvi20080214Brian P. Walenz A kmer/libkmer/existDB.C jcvi20071029Brian P. Walenz A kmer/libkmer/existDB.C jcvi20071010Brian P. Walenz A kmer/libkmer/existDB.C jcvi20070608Brian P. Walenz A kmer/libkmer/existDB.C jcvi20070607Brian P. Walenz A kmer/libkmer/existDB.C jcvi20070326Brian P. Walenz A kmer/libkmer/existDB.C jcvi20060718Brian P. Walenz A kmer/libkmer/existDB.C jcvi20051204Brian P. Walenz A kmer/libkmer/existDB.C jcvi20051127Brian P. Walenz A kmer/libkmer/existDB.C jcvi20051127Brian P. Walenz A kmer/libkmer/existDB.C jcvi20050320Brian P. Walenz A kmer/libkmer/existDB.C none20041010Brian P. Walenz A kmer/libkmer/existDB.C none20040421Brian P. Walenz A kmer/libkmer/existDB.C none20040412Brian P. Walenz A kmer/libkmer/existDB.C craa20031020Brian P. Walenz A kmer/libkmer/existDB.C craa20031016Brian P. Walenz A kmer/libkmer/existDB.C craa20030908Brian P. Walenz A kmer/libkmer/existDB.C craa20030805Brian P. Walenz A kmer/libkmer/existDB.C craa20030220Brian P. Walenz A kmer/libkmer/existDB.C craa20030213Brian P. Walenz A kmer/libkmer/existDB.C craa20030213Brian P. Walenz A kmer/libkmer/existDB.C craa20030102Brian P. Walenz A src/overlapInCore-analysis/filterTrue.pl nihh20151012Brian P. Walenz A src/overlapInCore-analysis/find-missed-true-overlaps.pl nihh20151012Brian P. Walenz A src/overlapInCore-analysis/infer-ovl-from-genomic-blasr.pl nihh20151012Brian P. Walenz A src/overlapInCore-analysis/check-ordered-reads-for-missed-overlaps.pl nihh20151012Brian P. Walenz A src/overlapInCore-analysis/infer-obt-from-genomic-blasr.pl nihh20151012Brian P. Walenz A src/overlapInCore-analysis/analyze-true-vs-test.pl nihh20151012Brian P. Walenz A src/overlapInCore-analysis/infer-olaps-from-pairwise-coords.pl nihh20151012Brian P. Walenz A src/overlapInCore-analysis/infer-olaps-from-pairwise-blasr.pl nihh20151012Brian P. Walenz A src/overlapInCore-analysis/infer-olaps-from-genomic-coords.pl nihh20151012Brian P. Walenz A src/bogart/AS_BAT_RepeatJunctionEvidence.H bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_RepeatJunctionEvidence.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_RepeatJunctionEvidence.H jcvi20130905Brian P. Walenz A src/bogart/AS_BAT_RepeatJunctionEvidence.H jcvi20130905Brian P. Walenz A src/bogart/AS_BAT_RepeatJunctionEvidence.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_RepeatJunctionEvidence.H jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_RepeatJunctionEvidence.H jcvi20120214Brian P. Walenz D src/bogart/AS_BAT_RepeatJunctionEvidence.H src/AS_BAT/AS_BAT_RepeatJunctionEvidence.H A src/bogart/AS_BAT_InsertSizes.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_InsertSizes.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_InsertSizes.H jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_InsertSizes.H jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_InsertSizes.H src/AS_BAT/AS_BAT_InsertSizes.H A src/bogart/AS_BAT_Unitig_AddAndPlaceFrag.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddAndPlaceFrag.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddAndPlaceFrag.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddAndPlaceFrag.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddAndPlaceFrag.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_Unitig_AddAndPlaceFrag.C src/AS_BAT/AS_BAT_Unitig_AddAndPlaceFrag.C A src/bogart/AS_BAT_PlaceContains.H bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_PlaceContains.H src/AS_BAT/AS_BAT_PlaceContains.H A src/bogart/AS_BAT_Joining.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Joining.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Joining.C jcvi20130430Brian P. Walenz A src/bogart/AS_BAT_Joining.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_Joining.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_Joining.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_Joining.C src/AS_BAT/AS_BAT_Joining.C A src/bogart/AS_BAT_Datatypes.H bnbi20150616Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H bnbi20150127Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H bnbi20141117Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H bnbi20140811Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20140128Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20130617Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20121211Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20120727Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20120321Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20120222Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20111229Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20111205Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20111007Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20110901Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20110404Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20101217Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_Datatypes.H src/AS_BAT/AS_BAT_Datatypes.H A src/bogart/AS_BAT_SetParentAndHang.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_SetParentAndHang.H src/AS_BAT/AS_BAT_SetParentAndHang.H A src/bogart/AS_BAT_Breaking.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Breaking.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Breaking.H jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_Breaking.H jcvi20110317Brian P. Walenz A src/bogart/AS_BAT_Breaking.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_Breaking.H src/AS_BAT/AS_BAT_Breaking.H A src/bogart/AS_BAT_PlaceZombies.C bnbi20150424Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.C jcvi20120222Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.C jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.C jcvi20110118Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_PlaceZombies.C src/AS_BAT/AS_BAT_PlaceZombies.C A src/bogart/AS_BAT_MoveContains.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_MoveContains.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_MoveContains.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_MoveContains.H src/AS_BAT/AS_BAT_MoveContains.H A src/bogart/AS_BAT_SplitDiscontinuous.C bnbi20150303Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C jcvi20120313Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C jcvi20111218Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C jcvi20110102Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_SplitDiscontinuous.C src/AS_BAT/AS_BAT_SplitDiscontinuous.C A src/bogart/AS_BAT_MateChecker.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_MateChecker.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_MateChecker.H jcvi20110317Brian P. Walenz A src/bogart/AS_BAT_MateChecker.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_MateChecker.H src/AS_BAT/AS_BAT_MateChecker.H A src/bogart/addReadsToUnitigs.C bnbi20150410Brian P. Walenz A src/bogart/addReadsToUnitigs.C bnbi20141009Brian P. Walenz A src/bogart/addReadsToUnitigs.C jcvi20140422Brian P. Walenz A src/bogart/addReadsToUnitigs.C jcvi20140331Brian P. Walenz A src/bogart/addReadsToUnitigs.C jcvi20130824Brian P. Walenz A src/bogart/addReadsToUnitigs.C jcvi20130824Brian P. Walenz D src/bogart/addReadsToUnitigs.C src/AS_CNS/addReadsToUnitigs.C A src/bogart/AS_BAT_Outputs.H bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_Outputs.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Outputs.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Outputs.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Outputs.H jcvi20120115Brian P. Walenz A src/bogart/AS_BAT_Outputs.H jcvi20111229Brian P. Walenz A src/bogart/AS_BAT_Outputs.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_Outputs.H src/AS_BAT/AS_BAT_Outputs.H A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C bnbi20150805Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C bnbi20150520Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C bnbi20141021Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C bnbi20141009Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20130908Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20130627Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20130627Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20130429Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20120214Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20111205Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20110408Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20110406Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20110404Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20110317Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20101217Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20101215Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20101204Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_PlaceFragUsingOverlaps.C src/AS_BAT/AS_BAT_PlaceFragUsingOverlaps.C A src/bogart/AS_BAT_Logging.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Logging.H bnbi20140811Brian P. Walenz A src/bogart/AS_BAT_Logging.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Logging.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Logging.H jcvi20130430Brian P. Walenz A src/bogart/AS_BAT_Logging.H jcvi20120729Brian P. Walenz D src/bogart/AS_BAT_Logging.H src/AS_BAT/AS_BAT_Logging.H A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20150814Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20150729Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20150520Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20150514Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20150424Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20150420Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20150303Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20141021Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C bnbi20141009Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20140129Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20140128Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20130627Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20130626Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20130430Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20121120Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20120730Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20120214Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20111218Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20111212Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20111206Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_BestOverlapGraph.C src/AS_BAT/AS_BAT_BestOverlapGraph.C A src/bogart/AS_BAT_ChunkGraph.C bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.C jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_ChunkGraph.C src/AS_BAT/AS_BAT_ChunkGraph.C A src/bogart/AS_BAT_EvaluateMates.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_EvaluateMates.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_EvaluateMates.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_EvaluateMates.C jcvi20111229Brian P. Walenz A src/bogart/AS_BAT_EvaluateMates.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_EvaluateMates.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_EvaluateMates.C src/AS_BAT/AS_BAT_EvaluateMates.C A src/bogart/AS_BAT_MateLocation.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_MateLocation.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_MateLocation.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_MateLocation.C jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_MateLocation.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_MateLocation.C jcvi20101202Brian P. Walenz A src/bogart/AS_BAT_MateLocation.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_MateLocation.C src/AS_BAT/AS_BAT_MateLocation.C A src/bogart/AS_BAT_IntersectSplit.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.C jcvi20110317Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.C jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.C jcvi20101213Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_IntersectSplit.C src/AS_BAT/AS_BAT_IntersectSplit.C A src/bogart/AS_BAT_Unitig.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Unitig.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Unitig.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_Unitig.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_Unitig.C jcvi20120115Brian P. Walenz A src/bogart/AS_BAT_Unitig.C jcvi20111229Brian P. Walenz A src/bogart/AS_BAT_Unitig.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_Unitig.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_Unitig.C src/AS_BAT/AS_BAT_Unitig.C A src/bogart/AS_BAT_PopulateUnitig.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_PopulateUnitig.H src/AS_BAT/AS_BAT_PopulateUnitig.H A src/bogart/AS_BAT_Instrumentation.C bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C jcvi20130827Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C jcvi20110728Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_Instrumentation.C src/AS_BAT/AS_BAT_Instrumentation.C A src/bogart/AS_BAT_OverlapCache.H bnbi20150616Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H bnbi20150520Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H bnbi20150514Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H bnbi20150424Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H bnbi20150303Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H bnbi20141021Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20130828Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20130626Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20120222Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20120215Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20111212Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20111209Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20111205Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20111106Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H bnbi20111013Sergey Koren A src/bogart/AS_BAT_OverlapCache.H bnbi20111013Sergey Koren A src/bogart/AS_BAT_OverlapCache.H jcvi20110803Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20110404Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H jcvi20110215Brian P. Walenz D src/bogart/AS_BAT_OverlapCache.H src/AS_BAT/AS_BAT_OverlapCache.H A src/bogart/AS_BAT_MergeSplitJoin.H bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H bnbi20150424Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H bnbi20150303Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H jcvi20120222Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H jcvi20111218Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H jcvi20110215Brian P. Walenz D src/bogart/AS_BAT_MergeSplitJoin.H src/AS_BAT/AS_BAT_MergeSplitJoin.H A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C bnbi20150306Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C bnbi20150127Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C jcvi20130626Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C jcvi20130626Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C jcvi20130429Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C jcvi20130429Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C jcvi20111213Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C jcvi20110317Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C jcvi20101204Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C src/AS_BAT/AS_BAT_Unitig_PlaceFragUsingEdges.C A src/bogart/AS_BAT_IntersectBubble.H bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.H jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_IntersectBubble.H src/AS_BAT/AS_BAT_IntersectBubble.H A src/bogart/AS_BAT_PlaceZombies.H bnbi20150424Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.H jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.H jcvi20110118Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_PlaceZombies.H src/AS_BAT/AS_BAT_PlaceZombies.H A src/bogart/AS_BAT_MoveContains.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_MoveContains.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_MoveContains.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_MoveContains.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_MoveContains.C jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_MoveContains.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_MoveContains.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_MoveContains.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_MoveContains.C src/AS_BAT/AS_BAT_MoveContains.C A src/bogart/AS_BAT_SplitDiscontinuous.H bnbi20150303Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_SplitDiscontinuous.H src/AS_BAT/AS_BAT_SplitDiscontinuous.H A src/bogart/AS_BAT_MateChecker.C nihh20151012Brian P. Walenz A src/bogart/AS_BAT_MateChecker.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_MateChecker.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_MateChecker.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_MateChecker.C jcvi20110317Brian P. Walenz A src/bogart/AS_BAT_MateChecker.C bnbi20110308Sergey Koren A src/bogart/AS_BAT_MateChecker.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_MateChecker.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_MateChecker.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_MateChecker.C src/AS_BAT/AS_BAT_MateChecker.C A src/bogart/AS_BAT_Joining.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Joining.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Joining.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_Joining.H src/AS_BAT/AS_BAT_Joining.H A src/bogart/AS_BAT_SetParentAndHang.C bnbi20150805Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C jcvi20101217Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_SetParentAndHang.C src/AS_BAT/AS_BAT_SetParentAndHang.C A src/bogart/AS_BAT_Breaking.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Breaking.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Breaking.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_Breaking.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_Breaking.C jcvi20120214Brian P. Walenz A src/bogart/AS_BAT_Breaking.C jcvi20110506Brian P. Walenz A src/bogart/AS_BAT_Breaking.C jcvi20110317Brian P. Walenz A src/bogart/AS_BAT_Breaking.C jcvi20110102Brian P. Walenz A src/bogart/AS_BAT_Breaking.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_Breaking.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_Breaking.C src/AS_BAT/AS_BAT_Breaking.C A src/bogart/bogart.C bnbi20150807Brian P. Walenz A src/bogart/bogart.C bnbi20150616Brian P. Walenz A src/bogart/bogart.C bnbi20150603Brian P. Walenz A src/bogart/bogart.C bnbi20150520Brian P. Walenz A src/bogart/bogart.C bnbi20150424Brian P. Walenz A src/bogart/bogart.C bnbi20150303Brian P. Walenz A src/bogart/bogart.C bnbi20141223Brian P. Walenz A src/bogart/bogart.C bnbi20141219Brian P. Walenz A src/bogart/bogart.C bnbi20141021Brian P. Walenz A src/bogart/bogart.C jcvi20140129Brian P. Walenz A src/bogart/bogart.C jcvi20131014Brian P. Walenz A src/bogart/bogart.C jcvi20130905Brian P. Walenz A src/bogart/bogart.C jcvi20130828Brian P. Walenz A src/bogart/bogart.C jcvi20130828Brian P. Walenz A src/bogart/bogart.C jcvi20130801Brian P. Walenz A src/bogart/bogart.C jcvi20130627Brian P. Walenz A src/bogart/bogart.C jcvi20130617Brian P. Walenz A src/bogart/bogart.C jcvi20130430Brian P. Walenz A src/bogart/bogart.C jcvi20120730Brian P. Walenz A src/bogart/bogart.C jcvi20120730Brian P. Walenz A src/bogart/bogart.C jcvi20120729Brian P. Walenz A src/bogart/bogart.C jcvi20120728Brian P. Walenz A src/bogart/bogart.C jcvi20120502Brian P. Walenz A src/bogart/bogart.C jcvi20120313Brian P. Walenz A src/bogart/bogart.C jcvi20120222Brian P. Walenz A src/bogart/bogart.C jcvi20120215Brian P. Walenz A src/bogart/bogart.C jcvi20120215Brian P. Walenz A src/bogart/bogart.C jcvi20120214Brian P. Walenz A src/bogart/bogart.C jcvi20120115Brian P. Walenz A src/bogart/bogart.C jcvi20120105Brian P. Walenz A src/bogart/bogart.C jcvi20111229Brian P. Walenz A src/bogart/bogart.C jcvi20111218Brian P. Walenz A src/bogart/bogart.C jcvi20111212Brian P. Walenz A src/bogart/bogart.C jcvi20111205Brian P. Walenz A src/bogart/bogart.C jcvi20110404Brian P. Walenz A src/bogart/bogart.C jcvi20110317Brian P. Walenz A src/bogart/bogart.C jcvi20110215Brian P. Walenz A src/bogart/bogart.C jcvi20110118Brian P. Walenz A src/bogart/bogart.C jcvi20110102Brian P. Walenz A src/bogart/bogart.C jcvi20101217Brian P. Walenz A src/bogart/bogart.C jcvi20101206Brian P. Walenz A src/bogart/bogart.C jcvi20101204Brian P. Walenz A src/bogart/bogart.C jcvi20101123Brian P. Walenz D src/bogart/bogart.C src/AS_BAT/bogart.C A src/bogart/AS_BAT_PlaceContains.C bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C bnbi20150127Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C jcvi20140129Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C jcvi20110417Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_PlaceContains.C src/AS_BAT/AS_BAT_PlaceContains.C A src/bogart/AS_BAT_InsertSizes.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_InsertSizes.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_InsertSizes.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_InsertSizes.C jcvi20111216Brian P. Walenz A src/bogart/AS_BAT_InsertSizes.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_InsertSizes.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_InsertSizes.C src/AS_BAT/AS_BAT_InsertSizes.C A src/bogart/AS_BAT_MateBubble.C bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_MateBubble.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_MateBubble.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_MateBubble.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_MateBubble.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_MateBubble.C jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_MateBubble.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_MateBubble.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_MateBubble.C src/AS_BAT/AS_BAT_MateBubble.C A src/bogart/AS_BAT_Instrumentation.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_Instrumentation.H src/AS_BAT/AS_BAT_Instrumentation.H A src/bogart/AS_BAT_OverlapCache.C bnbi20150625Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20150616Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20150520Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20150514Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20150424Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20150303Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20150204Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20150113Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20141021Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20140806Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20131014Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20130929Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20130905Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20130828Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20130626Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20120727Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20120412Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20120321Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20120222Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20120218Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20120215Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20120215Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20120114Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C bnbi20120111Sergey Koren A src/bogart/AS_BAT_OverlapCache.C jcvi20111229Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20111229Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20111212Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20111209Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20111209Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20111206Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20111205Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20111205Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20110831Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20110417Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20110404Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C jcvi20110215Brian P. Walenz D src/bogart/AS_BAT_OverlapCache.C src/AS_BAT/AS_BAT_OverlapCache.C A src/bogart/AS_BAT_ExtendByMates.C bnbi20150424Brian P. Walenz A src/bogart/AS_BAT_ExtendByMates.C bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_ExtendByMates.C bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_ExtendByMates.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_ExtendByMates.C jcvi20140128Brian P. Walenz A src/bogart/AS_BAT_ExtendByMates.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_ExtendByMates.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_ExtendByMates.C jcvi20120105Brian P. Walenz D src/bogart/AS_BAT_ExtendByMates.C src/AS_BAT/AS_BAT_ExtendByMates.C A src/bogart/AS_BAT_MergeSplitJoin.C bnbi20150805Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C bnbi20150303Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C bnbi20141014Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C bnbi20141009Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20140503Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20140304Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20130908Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20130905Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20130828Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20130430Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20130429Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20120806Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20120730Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20120730Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20120313Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20120222Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20120214Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20111218Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20111213Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20110524Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20110506Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20110417Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20110408Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20110406Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20110404Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20110317Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C jcvi20110215Brian P. Walenz D src/bogart/AS_BAT_MergeSplitJoin.C src/AS_BAT/AS_BAT_MergeSplitJoin.C A src/bogart/AS_BAT_IntersectBubble.C bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20110404Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20110118Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20110106Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20101215Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20101204Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_IntersectBubble.C src/AS_BAT/AS_BAT_IntersectBubble.C A src/bogart/AS_BAT_EvaluateMates.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_EvaluateMates.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_EvaluateMates.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_EvaluateMates.H src/AS_BAT/AS_BAT_EvaluateMates.H A src/bogart/AS_BAT_MateLocation.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_MateLocation.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_MateLocation.H jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_MateLocation.H jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_MateLocation.H src/AS_BAT/AS_BAT_MateLocation.H A src/bogart/AS_BAT_IntersectSplit.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.H jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_IntersectSplit.H src/AS_BAT/AS_BAT_IntersectSplit.H A src/bogart/AS_BAT_Unitig.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Unitig.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Unitig.H jcvi20130626Brian P. Walenz A src/bogart/AS_BAT_Unitig.H jcvi20130318Brian P. Walenz A src/bogart/AS_BAT_Unitig.H jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_Unitig.H jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_Unitig.H jcvi20120214Brian P. Walenz A src/bogart/AS_BAT_Unitig.H jcvi20120115Brian P. Walenz A src/bogart/AS_BAT_Unitig.H jcvi20111229Brian P. Walenz A src/bogart/AS_BAT_Unitig.H jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_Unitig.H jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_Unitig.H src/AS_BAT/AS_BAT_Unitig.H A src/bogart/AS_BAT_PopulateUnitig.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C jcvi20110901Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_PopulateUnitig.C src/AS_BAT/AS_BAT_PopulateUnitig.C A src/bogart/AS_BAT_PromoteToSingleton.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_PromoteToSingleton.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_PromoteToSingleton.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_PromoteToSingleton.C jcvi20120728Brian P. Walenz A src/bogart/AS_BAT_PromoteToSingleton.C jcvi20120105Brian P. Walenz D src/bogart/AS_BAT_PromoteToSingleton.C src/AS_BAT/AS_BAT_PromoteToSingleton.C A src/bogart/AS_BAT_Unitig_AddFrag.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddFrag.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddFrag.C jcvi20130626Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddFrag.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddFrag.C jcvi20120214Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddFrag.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddFrag.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_Unitig_AddFrag.C src/AS_BAT/AS_BAT_Unitig_AddFrag.C A src/bogart/AS_BAT_FragmentInfo.C bnbi20150616Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C bnbi20150529Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C bnbi20150317Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C jcvi20130617Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C jcvi20120806Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C jcvi20110417Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_FragmentInfo.C src/AS_BAT/AS_BAT_FragmentInfo.C A src/bogart/AS_BAT_findEdges.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_findEdges.C bnbi20141114Brian P. Walenz D src/bogart/AS_BAT_findEdges.C src/AS_BAT/AS_BAT_findEdges.C A src/bogart/AS_BAT_Outputs.C bnbi20150605Brian P. Walenz A src/bogart/AS_BAT_Outputs.C bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_Outputs.C bnbi20150409Brian P. Walenz A src/bogart/AS_BAT_Outputs.C bnbi20150127Brian P. Walenz A src/bogart/AS_BAT_Outputs.C bnbi20150113Brian P. Walenz A src/bogart/AS_BAT_Outputs.C bnbi20150113Brian P. Walenz A src/bogart/AS_BAT_Outputs.C bnbi20141223Brian P. Walenz A src/bogart/AS_BAT_Outputs.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Outputs.C bnbi20141117Brian P. Walenz A src/bogart/AS_BAT_Outputs.C jcvi20140331Brian P. Walenz A src/bogart/AS_BAT_Outputs.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Outputs.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Outputs.C jcvi20120115Brian P. Walenz A src/bogart/AS_BAT_Outputs.C jcvi20111229Brian P. Walenz A src/bogart/AS_BAT_Outputs.C jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_Outputs.C jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_Outputs.C src/AS_BAT/AS_BAT_Outputs.C A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H jcvi20120214Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H jcvi20110408Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H jcvi20110406Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H jcvi20110404Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H jcvi20110317Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H jcvi20110118Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H jcvi20101215Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H jcvi20101206Brian P. Walenz D src/bogart/AS_BAT_PlaceFragUsingOverlaps.H src/AS_BAT/AS_BAT_PlaceFragUsingOverlaps.H A src/bogart/AS_BAT_Logging.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_Logging.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_Logging.C jcvi20130617Brian P. Walenz A src/bogart/AS_BAT_Logging.C jcvi20130430Brian P. Walenz A src/bogart/AS_BAT_Logging.C jcvi20120729Brian P. Walenz D src/bogart/AS_BAT_Logging.C src/AS_BAT/AS_BAT_Logging.C A src/bogart/AS_BAT_BestOverlapGraph.H bnbi20150603Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H bnbi20150424Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H bnbi20141021Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H jcvi20140129Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H jcvi20140128Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H jcvi20130430Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H jcvi20120214Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H jcvi20111205Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H jcvi20110215Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H jcvi20101206Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_BestOverlapGraph.H src/AS_BAT/AS_BAT_BestOverlapGraph.H A src/bogart/AS_BAT_ChunkGraph.H bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.H bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.H jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.H jcvi20120105Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.H jcvi20101123Brian P. Walenz D src/bogart/AS_BAT_ChunkGraph.H src/AS_BAT/AS_BAT_ChunkGraph.H A src/bogart/AS_BAT_ReconstructRepeats.C bnbi20150424Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C bnbi20141222Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C bnbi20141219Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C jcvi20130801Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C jcvi20120729Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C jcvi20120214Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C jcvi20120105Brian P. Walenz D src/bogart/AS_BAT_ReconstructRepeats.C src/AS_BAT/AS_BAT_ReconstructRepeats.C A src/bogart/analyzeBest.C bnbi20150410Brian P. Walenz A src/bogart/analyzeBest.C bnbi20141121Brian P. Walenz A src/bogart/analyzeBest.C bnbi20141009Brian P. Walenz A src/bogart/analyzeBest.C jcvi20130801Brian P. Walenz A src/bogart/analyzeBest.C jcvi20130801Brian P. Walenz A src/bogart/analyzeBest.C jcvi20130603Brian P. Walenz A src/bogart/analyzeBest.C jcvi20111229Brian P. Walenz A src/bogart/analyzeBest.C jcvi20101009Brian P. Walenz D src/bogart/analyzeBest.C src/AS_BOG/analyzeBest.C A src/overlapErrorAdjustment/findErrors.C bnbi20150701Brian P. Walenz A src/overlapErrorAdjustment/findErrors.C bnbi20150616Brian P. Walenz A src/overlapErrorAdjustment/findErrors.C bnbi20150603Brian P. Walenz A src/overlapErrorAdjustment/findErrors.C bnbi20150529Brian P. Walenz A src/overlapErrorAdjustment/analyzeAlignment.H bnbi20150701Brian P. Walenz A src/overlapErrorAdjustment/analyzeAlignment.H bnbi20150624Brian P. Walenz A src/overlapErrorAdjustment/analyzeAlignment.H bnbi20150623Brian P. Walenz A src/overlapErrorAdjustment/analyzeAlignment.H bnbi20150618Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps.C bnbi20150616Brian P. Walenz A src/overlapErrorAdjustment/findErrors-Read_Olaps.C bnbi20150625Brian P. Walenz A src/overlapErrorAdjustment/findErrors-Read_Olaps.C bnbi20150616Brian P. Walenz A src/overlapErrorAdjustment/findErrors-Read_Olaps.C bnbi20150616Brian P. Walenz A src/overlapErrorAdjustment/findErrors-Read_Frags.C bnbi20150603Brian P. Walenz A src/overlapErrorAdjustment/findErrors-Read_Frags.C bnbi20150529Brian P. Walenz A src/overlapErrorAdjustment/findErrors-Read_Frags.C bnbi20150505Brian P. Walenz A src/overlapErrorAdjustment/findErrors-Analyze_Alignment.C bnbi20150618Brian P. Walenz A src/overlapErrorAdjustment/analyzeAlignment.C bnbi20150701Brian P. Walenz A src/overlapErrorAdjustment/analyzeAlignment.C bnbi20150701Brian P. Walenz A src/overlapErrorAdjustment/analyzeAlignment.C bnbi20150624Brian P. Walenz A src/overlapErrorAdjustment/analyzeAlignment.C bnbi20150623Brian P. Walenz A src/overlapErrorAdjustment/analyzeAlignment.C bnbi20150618Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Redo_Olaps.C bnbi20150603Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Redo_Olaps.C bnbi20150529Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Redo_Olaps.C bnbi20150520Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Redo_Olaps.C bnbi20150514Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Read_Olaps.C bnbi20150625Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Read_Olaps.C bnbi20150616Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Read_Olaps.C bnbi20150616Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Correct_Frags.C bnbi20150603Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Correct_Frags.C bnbi20150529Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Correct_Frags.C bnbi20150520Brian P. Walenz A src/overlapInCore/overlapImport.C bnbi20150625Brian P. Walenz A src/overlapInCore/overlapImport.C bnbi20150616Brian P. Walenz A src/overlapInCore/overlapImport.C bnbi20150616Brian P. Walenz A src/overlapInCore/overlapImport.C bnbi20150529Brian P. Walenz A src/overlapInCore/overlapImport.C bnbi20150514Brian P. Walenz A src/overlapInCore/overlapReadCache.H bnbi20150623Brian P. Walenz A src/overlapInCore/overlapReadCache.H bnbi20150618Brian P. Walenz A src/overlapInCore/overlapReadCache.H bnbi20150616Brian P. Walenz A src/overlapInCore/overlapReadCache.H bnbi20150616Brian P. Walenz D src/overlapInCore/overlapReadCache.H src/overlapInCore/overlapPair-readCache.H A src/overlapInCore/overlapInCore-Output.C nihh20151026Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C bnbi20150616Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C bnbi20150409Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C bnbi20150321Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C bnbi20150317Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C bnbi20150211Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C bnbi20150113Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C bnbi20141215Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C bnbi20141117Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20130801Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20130501Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20111213Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20110801Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C bnbi20110617Sergey Koren A src/overlapInCore/overlapInCore-Output.C bnbi20110308Sergey Koren A src/overlapInCore/overlapInCore-Output.C jcvi20100830Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20100819Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20100402Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20100330Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20100216Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20091028Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20091026Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20090730Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20090610Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20090520Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20090202Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20090116Sergey Koren A src/overlapInCore/overlapInCore-Output.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20081007Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20080627Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20080515Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20080227Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20080225Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20071120Eli Venter A src/overlapInCore/overlapInCore-Output.C jcvi20071109Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20071108Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070827Sergey Koren A src/overlapInCore/overlapInCore-Output.C jcvi20070810Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070803Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070801Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070529Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070416Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070404Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070329Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070305Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070225Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070223Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070218Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070214Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070213Art Delcher A src/overlapInCore/overlapInCore-Output.C jcvi20070212Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070211Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070208Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070129Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20070128Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20060926Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20060925Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20060821Aaron Halpern A src/overlapInCore/overlapInCore-Output.C jcvi20060605Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20060327Aaron Halpern A src/overlapInCore/overlapInCore-Output.C jcvi20050915Eli Venter A src/overlapInCore/overlapInCore-Output.C jcvi20050824Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20050805Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20050804Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20050801Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20050715Eli Venter A src/overlapInCore/overlapInCore-Output.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20050711Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C jcvi20050616Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore-Output.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore-Output.C tigr20040923Michael Schatz D src/overlapInCore/overlapInCore-Output.C src/AS_OVL/AS_OVL_overlap_common.h D src/overlapInCore/overlapInCore-Output.C src/AS_OVM/overlapInCore-Output.C A src/overlapInCore/overlapInCore.H nihh20151027Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20150825Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20150616Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20150410Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20150317Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20150303Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20150211Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20150209Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20150206Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20150206Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20141219Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20141215Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20130801Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20130801Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20130617Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20130501Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20130501Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20130111Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20120203Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20111213Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20110801Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20110801Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore.H bnbi20110308Sergey Koren A src/overlapInCore/overlapInCore.H jcvi20100819Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20100330Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20100329Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20091026Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20090610Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20090202Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20081111Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20081007Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20080627Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20071108Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070803Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070801Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070723Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070404Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070404Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070329Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070223Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070218Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070213Art Delcher A src/overlapInCore/overlapInCore.H jcvi20070212Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070211Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070129Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20070128Brian P. Walenz A src/overlapInCore/overlapInCore.H jcvi20060823Aaron Halpern A src/overlapInCore/overlapInCore.H jcvi20060822Aaron Halpern A src/overlapInCore/overlapInCore.H jcvi20060821Aaron Halpern A src/overlapInCore/overlapInCore.H jcvi20060327Aaron Halpern A src/overlapInCore/overlapInCore.H jcvi20050929Aaron Halpern A src/overlapInCore/overlapInCore.H jcvi20050616Brian P. Walenz A src/overlapInCore/overlapInCore.H tigr20050322Jason Miller A src/overlapInCore/overlapInCore.H tigr20050322Jason Miller A src/overlapInCore/overlapInCore.H tigr20040923Michael Schatz D src/overlapInCore/overlapInCore.H src/AS_OVL/AS_OVL_overlap.h D src/overlapInCore/overlapInCore.H src/AS_OVM/overlapInCore.H A src/overlapInCore/overlapInCorePartition.C bnbi20150825Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C bnbi20150616Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C bnbi20150529Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C bnbi20150317Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C bnbi20150227Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C bnbi20150223Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C bnbi20150206Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C bnbi20141121Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C bnbi20131202Sergey Koren A src/overlapInCore/overlapInCorePartition.C bnbi20131022Sergey Koren A src/overlapInCore/overlapInCorePartition.C jcvi20130801Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20130801Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C bnbi20130627Sergey Koren A src/overlapInCore/overlapInCorePartition.C bnbi20120729Sergey Koren A src/overlapInCore/overlapInCorePartition.C jcvi20120714Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20120220Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20120212Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20120211Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20120206Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20120206Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20120130Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20110706Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20110705Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20110627Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C jcvi20110612Brian P. Walenz D src/overlapInCore/overlapInCorePartition.C src/AS_OVL/overlap_partition.C A src/overlapInCore/overlapInCore-Process_String_Overlaps.C nihh20151029Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C bnbi20150720Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C bnbi20150720Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C bnbi20150708Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C bnbi20150410Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C bnbi20150331Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C bnbi20150211Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C bnbi20141215Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20130801Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20130501Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20130111Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20111213Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20110801Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C bnbi20110617Sergey Koren A src/overlapInCore/overlapInCore-Process_String_Overlaps.C bnbi20110308Sergey Koren A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20100830Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20100819Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20100402Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20100330Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20100216Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20091028Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20091026Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20090730Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20090610Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20090520Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20090202Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20090116Sergey Koren A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20081007Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20080627Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20080515Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20080227Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20080225Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20071120Eli Venter A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20071109Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20071108Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070827Sergey Koren A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070810Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070803Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070801Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070529Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070416Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070404Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070329Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070305Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070225Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070223Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070218Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070214Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070213Art Delcher A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070212Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070211Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070208Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070129Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20070128Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20060926Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20060925Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20060821Aaron Halpern A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20060605Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20060327Aaron Halpern A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050915Eli Venter A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050824Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050805Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050804Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050801Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050715Eli Venter A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050711Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C jcvi20050616Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore-Process_String_Overlaps.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore-Process_String_Overlaps.C tigr20040923Michael Schatz D src/overlapInCore/overlapInCore-Process_String_Overlaps.C src/AS_OVL/AS_OVL_overlap_common.h D src/overlapInCore/overlapInCore-Process_String_Overlaps.C src/AS_OVM/overlapInCore-Process_String_Overlaps.C A src/overlapInCore/overlapPair.C bnbi20150720Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150717Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150709Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150708Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150701Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150701Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150625Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150623Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150623Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150618Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150617Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150616Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150616Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150603Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150410Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150409Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150331Brian P. Walenz A src/overlapInCore/overlapPair.C bnbi20150327Brian P. Walenz A src/overlapInCore/overlapReadCache.C bnbi20150701Brian P. Walenz A src/overlapInCore/overlapReadCache.C bnbi20150625Brian P. Walenz A src/overlapInCore/overlapConvert.C bnbi20150625Brian P. Walenz A src/overlapInCore/overlapConvert.C bnbi20150616Brian P. Walenz A src/overlapInCore/overlapConvert.C bnbi20150616Brian P. Walenz A src/overlapInCore/overlapConvert.C bnbi20150605Brian P. Walenz A src/overlapInCore/overlapConvert.C bnbi20150514Brian P. Walenz A src/overlapInCore/overlapConvert.C bnbi20150429Brian P. Walenz A src/overlapInCore/overlapConvert.C bnbi20150409Brian P. Walenz A src/overlapInCore/overlapConvert.C bnbi20150321Brian P. Walenz A src/overlapInCore/overlapConvert.C bnbi20150317Brian P. Walenz A src/overlapInCore/overlapConvert.C bnbi20150211Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20150825Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20150720Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20150720Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20150701Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20150603Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20150529Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20150317Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20150227Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20141219Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20141215Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20141215Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20130801Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20130501Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20120130Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20111229Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20111213Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20110801Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20110801Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20110617Sergey Koren A src/overlapInCore/overlapInCore-Build_Hash_Index.C bnbi20110308Sergey Koren A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20100830Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20100819Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20100402Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20100330Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20100216Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20091028Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20091026Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20090730Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20090610Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20090520Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20090202Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20090116Sergey Koren A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20081007Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20080627Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20080515Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20080227Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20080225Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20071120Eli Venter A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20071109Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20071108Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070827Sergey Koren A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070810Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070803Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070801Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070529Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070416Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070404Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070329Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070305Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070225Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070223Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070218Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070214Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070213Art Delcher A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070212Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070211Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070208Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070129Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20070128Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20060926Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20060925Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20060821Aaron Halpern A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20060605Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20060327Aaron Halpern A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050915Eli Venter A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050824Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050805Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050804Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050801Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050715Eli Venter A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050711Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C jcvi20050616Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore-Build_Hash_Index.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore-Build_Hash_Index.C tigr20040923Michael Schatz D src/overlapInCore/overlapInCore-Build_Hash_Index.C src/AS_OVL/AS_OVL_overlap_common.h D src/overlapInCore/overlapInCore-Build_Hash_Index.C src/AS_OVM/overlapInCore-Build_Hash_Index.C A src/overlapInCore/overlapInCore-Find_Overlaps.C bnbi20141215Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20130801Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20130617Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20130501Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20111213Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20110801Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C bnbi20110617Sergey Koren A src/overlapInCore/overlapInCore-Find_Overlaps.C bnbi20110308Sergey Koren A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20100830Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20100819Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20100402Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20100330Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20100216Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20091028Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20091026Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20090730Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20090610Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20090520Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20090202Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20090116Sergey Koren A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20081007Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20080627Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20080515Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20080227Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20080225Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20071120Eli Venter A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20071109Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20071108Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070827Sergey Koren A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070810Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070803Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070801Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070529Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070416Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070404Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070329Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070305Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070225Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070223Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070218Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070214Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070213Art Delcher A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070212Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070211Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070208Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070129Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20070128Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20060926Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20060925Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20060821Aaron Halpern A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20060605Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20060327Aaron Halpern A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050915Eli Venter A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050824Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050805Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050804Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050801Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050715Eli Venter A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050711Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C jcvi20050616Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore-Find_Overlaps.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore-Find_Overlaps.C tigr20040923Michael Schatz D src/overlapInCore/overlapInCore-Find_Overlaps.C src/AS_OVL/AS_OVL_overlap_common.h D src/overlapInCore/overlapInCore-Find_Overlaps.C src/AS_OVM/overlapInCore-Find_Overlaps.C A src/overlapInCore/overlapInCore.C nihh20151027Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150825Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150720Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150720Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150708Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150701Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150625Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150616Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150603Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150602Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150317Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150303Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150211Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150209Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150206Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20150131Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20141219Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20141215Brian P. Walenz A src/overlapInCore/overlapInCore.C bnbi20140811Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20130801Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20130619Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20130617Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20130501Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20130111Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20120312Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20120212Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20120203Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20120130Brian P. Walenz A src/overlapInCore/overlapInCore.C tigr20120126Michael Schatz A src/overlapInCore/overlapInCore.C jcvi20111229Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20111213Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20111213Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20111111Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20110801Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20110801Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20090610Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20080627Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20071108Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070812Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070812Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070514Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070504Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070329Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070305Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070222Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070218Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070214Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070212Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070211Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070208Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070129Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070129Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070128Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20070126Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20060926Brian P. Walenz A src/overlapInCore/overlapInCore.C jcvi20050616Brian P. Walenz A src/overlapInCore/overlapInCore.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore.C tigr20040923Michael Schatz D src/overlapInCore/overlapInCore.C src/AS_OVL/AS_OVL_driver_common.h D src/overlapInCore/overlapInCore.C src/AS_OVM/overlapInCore.C A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1500.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1500.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1200.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1200.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2900.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2900.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2700.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2700.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2000.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2000.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3100.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3100.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/Binomial_Bound.C nihh20151012Brian P. Walenz A src/overlapInCore/liboverlap/Binomial_Bound.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/Binomial_Bound.C bnbi20150209Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3600.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3600.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/Display_Alignment.C bnbi20150708Brian P. Walenz A src/overlapInCore/liboverlap/Display_Alignment.C bnbi20150625Brian P. Walenz A src/overlapInCore/liboverlap/Display_Alignment.C bnbi20150624Brian P. Walenz A src/overlapInCore/liboverlap/Display_Alignment.C bnbi20150209Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3800.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3800.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4400.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4400.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4300.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4300.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0300.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0300.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0400.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0400.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimitGenerate.C bnbi20150814Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimitGenerate.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2100.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2100.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2600.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2600.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2800.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2800.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H nihh20151029Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H nihh20151026Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150720Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150720Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150717Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150713Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150708Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150605Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150410Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150303Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150303Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H bnbi20150211Brian P. Walenz D src/overlapInCore/liboverlap/prefixEditDistance.H src/overlapInCore/prefixEditDistance.H A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150720Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150720Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150717Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150713Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150709Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150709Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150708Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150708Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150701Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-forward.C bnbi20150211Brian P. Walenz D src/overlapInCore/liboverlap/prefixEditDistance-forward.C src/overlapInCore/prefixEditDistance-forward.C A src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C bnbi20150720Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C bnbi20150221Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C bnbi20150211Brian P. Walenz D src/overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C src/overlapInCore/prefixEditDistance-allocateMoreSpace.C A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1300.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1300.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1400.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1400.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0500.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0500.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0200.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0200.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C nihh20151029Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150720Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150720Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150713Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150708Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150708Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150701Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150624Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150617Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150410Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150331Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150211Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150211Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20150206Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20141215Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20130801Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20130501Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20130111Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20111213Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20110807Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20110803Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20110803Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20110801Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20110729Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20110729Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20110617Sergey Koren A src/overlapInCore/liboverlap/prefixEditDistance-extend.C bnbi20110308Sergey Koren A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20100830Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20100819Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20100402Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20100330Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20100216Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20091028Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20091026Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20090730Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20090610Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20090520Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20090202Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20090116Sergey Koren A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20081008Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20081008Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20081007Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20080627Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20080616Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20080616Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20080515Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20080227Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20080225Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20071120Eli Venter A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20071109Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20071108Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070827Sergey Koren A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070810Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070803Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070801Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070529Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070416Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070404Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070329Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070305Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070225Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070223Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070218Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070214Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070213Art Delcher A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070212Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070211Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070208Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070129Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20070128Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20060926Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20060925Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20060821Aaron Halpern A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20060605Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20060327Aaron Halpern A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050915Eli Venter A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050824Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050805Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050804Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050801Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050715Eli Venter A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050713Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050713Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050713Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050711Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C jcvi20050616Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C tigr20050322Jason Miller A src/overlapInCore/liboverlap/prefixEditDistance-extend.C tigr20050322Jason Miller A src/overlapInCore/liboverlap/prefixEditDistance-extend.C tigr20040923Michael Schatz D src/overlapInCore/liboverlap/prefixEditDistance-extend.C src/AS_OVL/AS_OVL_overlap_common.h D src/overlapInCore/liboverlap/prefixEditDistance-extend.C src/AS_OVM/overlapInCore-Extend_Alignment.C D src/overlapInCore/liboverlap/prefixEditDistance-extend.C src/overlapInCore/overlapInCore-Extend_Alignment.C D src/overlapInCore/liboverlap/prefixEditDistance-extend.C src/overlapInCore/prefixEditDistance-extend.C A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4200.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4200.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3900.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3900.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4500.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4500.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3700.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3700.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3000.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3000.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0100.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0100.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0600.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0600.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150720Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150720Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150717Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150713Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150709Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150709Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150708Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150708Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150701Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-reverse.C bnbi20150211Brian P. Walenz D src/overlapInCore/liboverlap/prefixEditDistance-reverse.C src/overlapInCore/prefixEditDistance-reverse.C A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0800.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0800.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/Binomial_Bound.H bnbi20150209Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4600.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4600.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4100.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4100.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/Display_Alignment.H bnbi20150708Brian P. Walenz A src/overlapInCore/liboverlap/Display_Alignment.H bnbi20150624Brian P. Walenz A src/overlapInCore/liboverlap/Display_Alignment.H bnbi20150209Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3300.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3300.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3400.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3400.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4800.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4800.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2500.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2500.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2200.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2200.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-5000.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-5000.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1900.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1900.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1700.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1700.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1000.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1000.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3500.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3500.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4900.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4900.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3200.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-3200.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4000.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4000.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4700.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-4700.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0900.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0900.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0700.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-0700.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1100.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1100.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1600.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1600.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1800.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-1800.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2300.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2300.C bnbi20150602Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C bnbi20150921Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C bnbi20150720Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C bnbi20150715Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C bnbi20150713Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C bnbi20150710Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C bnbi20150708Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C bnbi20150605Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2400.C bnbi20150603Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimit-2400.C bnbi20150602Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20150825Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20150824Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20150821Sergey Koren A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20150824Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20150720Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20150720Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20150603Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20150529Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20150317Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20150113Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20141219Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20141215Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20130801Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20111213Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20110803Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20110801Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20110729Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20110617Sergey Koren A src/overlapInCore/overlapInCore-Process_Overlaps.C bnbi20110308Sergey Koren A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20100830Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20100819Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20100402Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20100330Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20100216Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20091028Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20091026Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20090730Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20090610Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20090520Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20090202Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20090116Sergey Koren A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20081008Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20081007Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20080627Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20080616Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20080515Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20080227Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20080225Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20071120Eli Venter A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20071109Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20071108Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070827Sergey Koren A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070810Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070803Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070801Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070529Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070416Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070404Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070329Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070305Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070225Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070223Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070218Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070214Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070213Art Delcher A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070212Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070211Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070208Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070129Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20070128Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20060926Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20060925Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20060821Aaron Halpern A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20060605Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20060327Aaron Halpern A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050915Eli Venter A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050824Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050805Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050804Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050801Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050715Eli Venter A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050713Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050711Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C jcvi20050616Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore-Process_Overlaps.C tigr20050322Jason Miller A src/overlapInCore/overlapInCore-Process_Overlaps.C tigr20040923Michael Schatz D src/overlapInCore/overlapInCore-Process_Overlaps.C src/AS_OVL/AS_OVL_overlap_common.h D src/overlapInCore/overlapInCore-Process_Overlaps.C src/AS_OVM/overlapInCore-Process_Overlaps.C A src/AS_global.C nihh20151108Brian P. Walenz A src/AS_global.C bnbi20150303Brian P. Walenz A src/AS_global.C jcvi20130801Brian P. Walenz A src/AS_global.C jcvi20130801Brian P. Walenz A src/AS_global.C jcvi20130801Brian P. Walenz A src/AS_global.C jcvi20130417Brian P. Walenz A src/AS_global.C jcvi20130314Brian P. Walenz A src/AS_global.C jcvi20120529Brian P. Walenz A src/AS_global.C jcvi20120508Brian P. Walenz A src/AS_global.C jcvi20120508Brian P. Walenz A src/AS_global.C jcvi20120321Brian P. Walenz A src/AS_global.C jcvi20110125Brian P. Walenz A src/AS_global.C jcvi20091125Brian P. Walenz A src/AS_global.C jcvi20090501Brian P. Walenz A src/AS_global.C jcvi20090408Brian P. Walenz A src/AS_global.C jcvi20090306Sergey Koren A src/AS_global.C jcvi20090222Brian P. Walenz A src/AS_global.C jcvi20081113Brian P. Walenz A src/AS_global.C jcvi20081106Brian P. Walenz A src/AS_global.C jcvi20081009Brian P. Walenz A src/AS_global.C jcvi20081008Brian P. Walenz A src/AS_global.C jcvi20081008Brian P. Walenz A src/AS_global.C jcvi20080716Brian P. Walenz A src/AS_global.C jcvi20080716Brian P. Walenz A src/AS_global.C jcvi20080716Brian P. Walenz A src/AS_global.C jcvi20080627Brian P. Walenz A src/AS_global.C jcvi20070809Brian P. Walenz A src/AS_global.C jcvi20070804Brian P. Walenz A src/AS_global.C jcvi20070803Brian P. Walenz A src/mercy/mercy.C bnbi20141205Brian P. Walenz A src/mercy/mercy.C jcvi20140411Brian P. Walenz A src/mercy/mercy.C jcvi20130801Brian P. Walenz A src/mercy/mercy.C jcvi20130801Brian P. Walenz A src/mercy/mercy.C jcvi20110314Brian P. Walenz A src/mercy/mercy.C jcvi20090512Brian P. Walenz A src/mercy/mercy.C jcvi20081009Brian P. Walenz A src/mercy/mercy.C jcvi20081008Brian P. Walenz A src/mercy/mercy.C jcvi20080627Brian P. Walenz A src/mercy/mercy.C jcvi20070319Brian P. Walenz D src/mercy/mercy.C src/AS_MER/mercy.C A src/mercy/mercy-regions.C bnbi20141205Brian P. Walenz A src/mercy/mercy-regions.C bnbi20141014Brian P. Walenz A src/mercy/mercy-regions.C bnbi20141009Brian P. Walenz A src/mercy/mercy-regions.C jcvi20130801Brian P. Walenz A src/mercy/mercy-regions.C jcvi20130801Brian P. Walenz A src/mercy/mercy-regions.C jcvi20111229Brian P. Walenz A src/mercy/mercy-regions.C jcvi20101004Brian P. Walenz A src/mercy/mercy-regions.C jcvi20081009Brian P. Walenz A src/mercy/mercy-regions.C jcvi20081008Brian P. Walenz A src/mercy/mercy-regions.C jcvi20080627Brian P. Walenz A src/mercy/mercy-regions.C jcvi20070319Brian P. Walenz D src/mercy/mercy-regions.C src/AS_MER/mercy-regions.C A src/utgcns/utgcns.C nihh20151030Brian P. Walenz A src/utgcns/utgcns.C nihh20151012Brian P. Walenz A src/utgcns/utgcns.C nihh20151009Brian P. Walenz A src/utgcns/utgcns.C bnbi20150807Brian P. Walenz A src/utgcns/utgcns.C bnbi20150730Brian P. Walenz A src/utgcns/utgcns.C bnbi20150717Brian P. Walenz A src/utgcns/utgcns.C bnbi20150708Brian P. Walenz A src/utgcns/utgcns.C bnbi20150528Brian P. Walenz A src/utgcns/utgcns.C bnbi20150507Brian P. Walenz A src/utgcns/utgcns.C bnbi20150507Brian P. Walenz A src/utgcns/utgcns.C bnbi20150409Brian P. Walenz A src/utgcns/utgcns.C bnbi20150306Brian P. Walenz A src/utgcns/utgcns.C bnbi20150306Brian P. Walenz A src/utgcns/utgcns.C bnbi20150303Brian P. Walenz A src/utgcns/utgcns.C bnbi20150131Brian P. Walenz A src/utgcns/utgcns.C bnbi20150127Brian P. Walenz A src/utgcns/utgcns.C bnbi20150127Brian P. Walenz A src/utgcns/utgcns.C bnbi20150127Brian P. Walenz A src/utgcns/utgcns.C bnbi20150123Brian P. Walenz A src/utgcns/utgcns.C bnbi20150121Brian P. Walenz A src/utgcns/utgcns.C bnbi20150113Brian P. Walenz A src/utgcns/utgcns.C bnbi20150110Brian P. Walenz A src/utgcns/utgcns.C bnbi20141230Brian P. Walenz A src/utgcns/utgcns.C jcvi20140331Brian P. Walenz A src/utgcns/utgcns.C jcvi20140304Brian P. Walenz A src/utgcns/utgcns.C jcvi20140206Brian P. Walenz A src/utgcns/utgcns.C jcvi20131104Brian P. Walenz A src/utgcns/utgcns.C jcvi20131031Brian P. Walenz A src/utgcns/utgcns.C jcvi20131030Brian P. Walenz A src/utgcns/utgcns.C jcvi20131028Brian P. Walenz A src/utgcns/utgcns.C jcvi20131011Brian P. Walenz A src/utgcns/utgcns.C jcvi20130918Brian P. Walenz A src/utgcns/utgcns.C jcvi20130909Brian P. Walenz A src/utgcns/utgcns.C jcvi20130801Brian P. Walenz A src/utgcns/utgcns.C jcvi20130801Brian P. Walenz A src/utgcns/utgcns.C jcvi20120829Brian P. Walenz A src/utgcns/utgcns.C jcvi20120810Brian P. Walenz A src/utgcns/utgcns.C jcvi20120806Brian P. Walenz A src/utgcns/utgcns.C jcvi20111204Brian P. Walenz A src/utgcns/utgcns.C jcvi20111129Brian P. Walenz A src/utgcns/utgcns.C jcvi20110102Brian P. Walenz A src/utgcns/utgcns.C jcvi20100923Brian P. Walenz A src/utgcns/utgcns.C jcvi20091209Brian P. Walenz A src/utgcns/utgcns.C jcvi20091209Brian P. Walenz A src/utgcns/utgcns.C jcvi20091130Brian P. Walenz A src/utgcns/utgcns.C jcvi20091104Brian P. Walenz A src/utgcns/utgcns.C jcvi20091012Brian P. Walenz A src/utgcns/utgcns.C jcvi20091010Brian P. Walenz A src/utgcns/utgcns.C jcvi20091007Brian P. Walenz A src/utgcns/utgcns.C jcvi20091007Brian P. Walenz A src/utgcns/utgcns.C jcvi20091005Brian P. Walenz D src/utgcns/utgcns.C src/AS_CNS/utgcns.C A src/utgcns/libNNalign/NNalign.H bnbi20150810Brian P. Walenz A src/utgcns/libNNalign/NNalign.H bnbi20150617Brian P. Walenz A src/utgcns/libNNalign/NNalign.H bnbi20150410Brian P. Walenz A src/utgcns/libNNalign/NNalign.H bnbi20150113Brian P. Walenz A src/utgcns/libNNalign/NNalign.H bnbi20150110Brian P. Walenz A src/utgcns/libNNalign/NNalign.H bnbi20141230Brian P. Walenz A src/utgcns/libNNalign/NNalign.H bnbi20141223Brian P. Walenz A src/utgcns/libNNalign/NNalign.H bnbi20141223Brian P. Walenz A src/utgcns/libNNalign/NNalign.H bnbi20141223Brian P. Walenz A src/utgcns/libNNalign/NNalign.H bnbi20141223Brian P. Walenz D src/utgcns/libNNalign/NNalign.H src/alignment/AS_ALN_aligners.H D src/utgcns/libNNalign/NNalign.H src/alignment/aligners.H A src/utgcns/libNNalign/NNalgorithm.C bnbi20150810Brian P. Walenz A src/utgcns/libNNalign/NNalgorithm.C bnbi20150603Brian P. Walenz A src/utgcns/libNNalign/NNalgorithm.C bnbi20141223Brian P. Walenz A src/utgcns/libNNalign/NNalgorithm.C bnbi20141223Brian P. Walenz A src/utgcns/libNNalign/NNalgorithm.C bnbi20141223Brian P. Walenz D src/utgcns/libNNalign/NNalgorithm.C src/alignment/AS_ALN_bruteforcedp.C D src/utgcns/libNNalign/NNalgorithm.C src/alignment/brute-force-dp.C A src/utgcns/libNNalign/NNalign.C bnbi20150810Brian P. Walenz A src/utgcns/libNNalign/NNalign.C bnbi20150617Brian P. Walenz A src/utgcns/libNNalign/NNalign.C bnbi20150603Brian P. Walenz A src/utgcns/libNNalign/NNalign.C bnbi20150303Brian P. Walenz A src/utgcns/libNNalign/NNalign.C bnbi20150121Brian P. Walenz A src/utgcns/libNNalign/NNalign.C bnbi20150113Brian P. Walenz A src/utgcns/libNNalign/NNalign.C bnbi20141223Brian P. Walenz A src/utgcns/libNNalign/NNalign.C bnbi20141223Brian P. Walenz D src/utgcns/libNNalign/NNalign.C src/alignment/AS_ALN_forcns.C D src/utgcns/libNNalign/NNalign.C src/alignment/alignment-drivers.C A src/utgcns/stashContains.H bnbi20150509Brian P. Walenz A src/utgcns/stashContains.H bnbi20150409Brian P. Walenz A src/utgcns/libcns/abIDs.H bnbi20150127Brian P. Walenz A src/utgcns/libcns/abIDs.H bnbi20150112Brian P. Walenz A src/utgcns/libcns/abIDs.H bnbi20150110Brian P. Walenz A src/utgcns/libcns/abIDs.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abIDs.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abIDs.H bnbi20141230Brian P. Walenz A src/utgcns/libcns/abIDs.H bnbi20141223Brian P. Walenz A src/utgcns/libcns/abIDs.H bnbi20141117Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20130107Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111215Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111208Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111206Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111129Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111117Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20111115Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20110921Jason Miller A src/utgcns/libcns/abIDs.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20091026Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090925Sergey Koren A src/utgcns/libcns/abIDs.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090914Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090907Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090804Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090710Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090622Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090421Sergey Koren A src/utgcns/libcns/abIDs.H jcvi20090417Sergey Koren A src/utgcns/libcns/abIDs.H jcvi20090415Sergey Koren A src/utgcns/libcns/abIDs.H jcvi20090108Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20090105Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20081230Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20081218Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20081211Sergey Koren A src/utgcns/libcns/abIDs.H jcvi20081107Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20080924Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20080627Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20080609Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20080425Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20080213Eli Venter A src/utgcns/libcns/abIDs.H jcvi20080128Sergey Koren A src/utgcns/libcns/abIDs.H jcvi20071108Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20071025Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20070925Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20070918Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20070905Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20070514Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20070316Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20070225Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20070214Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20070212Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20070208Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20061005Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20061002Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20060923Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20060918Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20060906Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20060612Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20060213Eli Venter A src/utgcns/libcns/abIDs.H jcvi20051215Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20051127Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20051120Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20050829Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20050818Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20050805Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20050708Brian P. Walenz A src/utgcns/libcns/abIDs.H jcvi20050629Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20050614Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20050612Gennady Denisov A src/utgcns/libcns/abIDs.H jcvi20050523Gennady Denisov D src/utgcns/libcns/abIDs.H src/AS_CNS/MultiAlignment_CNS.h D src/utgcns/libcns/abIDs.H src/AS_CNS/MultiAlignment_CNS_private.H D src/utgcns/libcns/abIDs.H src/AS_CNS/MultiAlignment_CNS_private.h D src/utgcns/libcns/abIDs.H src/utgcns/libcns/MultiAlignment_CNS_private.H A src/utgcns/libcns/unitigConsensus.H nihh20151019Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H nihh20151013Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H nihh20151009Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150805Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150723Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150720Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150715Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150708Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150303Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150127Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150121Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150113Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150112Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150110Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20141230Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20141223Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H bnbi20141117Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20130107Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111215Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111208Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111206Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111129Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111117Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20111115Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20110921Jason Miller A src/utgcns/libcns/unitigConsensus.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20091026Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090925Sergey Koren A src/utgcns/libcns/unitigConsensus.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090914Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090907Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090804Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090710Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090622Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090421Sergey Koren A src/utgcns/libcns/unitigConsensus.H jcvi20090417Sergey Koren A src/utgcns/libcns/unitigConsensus.H jcvi20090415Sergey Koren A src/utgcns/libcns/unitigConsensus.H jcvi20090108Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20090105Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20081230Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20081218Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20081211Sergey Koren A src/utgcns/libcns/unitigConsensus.H jcvi20081107Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20080924Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20080627Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20080609Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20080425Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20080213Eli Venter A src/utgcns/libcns/unitigConsensus.H jcvi20080128Sergey Koren A src/utgcns/libcns/unitigConsensus.H jcvi20071108Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20071025Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20070925Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20070918Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20070905Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20070514Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20070316Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20070225Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20070214Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20070212Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20070208Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20061005Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20061002Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20060923Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20060918Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20060906Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20060612Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20060213Eli Venter A src/utgcns/libcns/unitigConsensus.H jcvi20051215Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20051127Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20051120Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20050829Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20050818Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20050805Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20050801Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20050801Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20050708Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H jcvi20050629Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20050614Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20050612Gennady Denisov A src/utgcns/libcns/unitigConsensus.H jcvi20050523Gennady Denisov D src/utgcns/libcns/unitigConsensus.H src/AS_CNS/MultiAlignment_CNS.h D src/utgcns/libcns/unitigConsensus.H src/AS_CNS/MultiAlignment_CNS_private.H D src/utgcns/libcns/unitigConsensus.H src/AS_CNS/MultiAlignment_CNS_private.h D src/utgcns/libcns/unitigConsensus.H src/utgcns/libcns/MultiAlignment_CNS_private.H A src/utgcns/libcns/abMultiAlign.H nihh20151014Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20150708Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20150127Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20150123Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20150113Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20150112Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20150110Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20150109Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20150108Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20141230Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20141223Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H bnbi20141117Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20130107Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111215Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111208Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111206Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111129Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111117Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20111115Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20110921Jason Miller A src/utgcns/libcns/abMultiAlign.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20091026Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090925Sergey Koren A src/utgcns/libcns/abMultiAlign.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090914Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090907Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090804Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090710Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090622Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090421Sergey Koren A src/utgcns/libcns/abMultiAlign.H jcvi20090417Sergey Koren A src/utgcns/libcns/abMultiAlign.H jcvi20090415Sergey Koren A src/utgcns/libcns/abMultiAlign.H jcvi20090108Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20090105Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20081230Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20081218Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20081211Sergey Koren A src/utgcns/libcns/abMultiAlign.H jcvi20081107Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20080924Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20080627Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20080609Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20080425Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20080213Eli Venter A src/utgcns/libcns/abMultiAlign.H jcvi20080128Sergey Koren A src/utgcns/libcns/abMultiAlign.H jcvi20071108Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20071025Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20070925Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20070918Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20070905Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20070514Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20070316Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20070225Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20070214Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20070212Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20070208Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20061005Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20061002Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20060923Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20060918Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20060906Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20060612Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20060213Eli Venter A src/utgcns/libcns/abMultiAlign.H jcvi20051215Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20051127Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20051120Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20050829Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20050818Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20050805Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20050708Brian P. Walenz A src/utgcns/libcns/abMultiAlign.H jcvi20050629Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20050614Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20050612Gennady Denisov A src/utgcns/libcns/abMultiAlign.H jcvi20050523Gennady Denisov D src/utgcns/libcns/abMultiAlign.H src/AS_CNS/MultiAlignment_CNS.h D src/utgcns/libcns/abMultiAlign.H src/AS_CNS/MultiAlignment_CNS_private.H D src/utgcns/libcns/abMultiAlign.H src/AS_CNS/MultiAlignment_CNS_private.h D src/utgcns/libcns/abMultiAlign.H src/utgcns/libcns/MultiAlignment_CNS_private.H A src/utgcns/libcns/abAbacus-refine.C nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C nihh20151014Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C nihh20151010Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C bnbi20150814Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C bnbi20150701Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C bnbi20150121Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C bnbi20150110Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C bnbi20150109Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C bnbi20141230Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111229Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111208Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20110921Jason Miller A src/utgcns/libcns/abAbacus-refine.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20100609Sergey Koren A src/utgcns/libcns/abAbacus-refine.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20091010Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090914Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090730Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090522Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090519Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090516Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090515Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090514Sergey Koren A src/utgcns/libcns/abAbacus-refine.C jcvi20090421Sergey Koren A src/utgcns/libcns/abAbacus-refine.C jcvi20090417Sergey Koren A src/utgcns/libcns/abAbacus-refine.C jcvi20090415Sergey Koren A src/utgcns/libcns/abAbacus-refine.C jcvi20090223Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090126Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090116Sergey Koren A src/utgcns/libcns/abAbacus-refine.C jcvi20090113Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090108Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090107Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090104Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20090102Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081219Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081205Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081113Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081112Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081107Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081105Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081008Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081007Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080925Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080731Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080627Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080606Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20080425Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080318Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080227Sergey Koren A src/utgcns/libcns/abAbacus-refine.C jcvi20080213Eli Venter A src/utgcns/libcns/abAbacus-refine.C jcvi20080204Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20080124Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20080101Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20071220Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20071217Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20071208Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20071108Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20071025Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20071003Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20071002Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070925Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070918Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070905Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070831Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070829Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070829Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070807Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070804Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070803Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070617Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070612Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070610Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070601Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070529Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070428Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070426Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070419Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070416Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070402Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070319Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070316Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070310Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20070305Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070304Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070226Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070225Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070218Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070214Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070212Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070208Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070204Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070203Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20070128Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20061212Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20061117Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-refine.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-refine.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061024Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061021Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061020Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061017Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061016Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061016Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20061015Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061013Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061010Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061009Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061008Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20061005Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061003Aaron Halpern A src/utgcns/libcns/abAbacus-refine.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060926Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060924Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060922Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060918Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060913Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060909Aaron Halpern A src/utgcns/libcns/abAbacus-refine.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060907Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060906Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060905Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060829Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060821Aaron Halpern A src/utgcns/libcns/abAbacus-refine.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060804Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060612Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060602Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060601Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060524Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060521Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060520Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060519Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060503Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060406Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060328Aaron Halpern A src/utgcns/libcns/abAbacus-refine.C jcvi20060324Eli Venter A src/utgcns/libcns/abAbacus-refine.C jcvi20060323Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060322Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060317Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060316Eli Venter A src/utgcns/libcns/abAbacus-refine.C jcvi20060308Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20060302Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060214Eli Venter A src/utgcns/libcns/abAbacus-refine.C jcvi20060130Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20060112Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20051216Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20051215Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20051213Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20051212Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20051212Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20051127Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20051120Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20051001Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-refine.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-refine.C jcvi20050926Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20050920Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20050918Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050917Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050914Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050912Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20050911Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050827Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050819Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20050819Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050818Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050812Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050810Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050808Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050805Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050727Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050708Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20050630Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050629Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050623Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050621Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050616Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C jcvi20050614Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050612Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050523Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050509Gennady Denisov A src/utgcns/libcns/abAbacus-refine.C jcvi20050502Eli Venter A src/utgcns/libcns/abAbacus-refine.C jcvi20050330Eli Venter A src/utgcns/libcns/abAbacus-refine.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-refine.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-refine.C tigr20040923Michael Schatz D src/utgcns/libcns/abAbacus-refine.C src/AS_CNS/AbacusRefine.C D src/utgcns/libcns/abAbacus-refine.C src/AS_CNS/AbacusRefine.c D src/utgcns/libcns/abAbacus-refine.C src/AS_CNS/MultiAlignment_CNS.c D src/utgcns/libcns/abAbacus-refine.C src/utgcns/libcns/AbacusRefine.C A src/utgcns/libcns/abAbacus.C nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus.C nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus.C nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus.C nihh20151014Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150708Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150306Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150123Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150121Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150121Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150113Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150112Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150110Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150109Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150108Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150108Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20141230Brian P. Walenz A src/utgcns/libcns/abAbacus.C bnbi20141117Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20091010Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090914Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090730Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090522Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090519Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090516Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090515Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090514Sergey Koren A src/utgcns/libcns/abAbacus.C jcvi20090421Sergey Koren A src/utgcns/libcns/abAbacus.C jcvi20090417Sergey Koren A src/utgcns/libcns/abAbacus.C jcvi20090415Sergey Koren A src/utgcns/libcns/abAbacus.C jcvi20090223Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090126Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090116Sergey Koren A src/utgcns/libcns/abAbacus.C jcvi20090113Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090108Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090107Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090104Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20090102Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081219Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081205Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081113Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081112Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081107Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081105Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081008Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081007Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080925Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080731Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080627Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080606Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20080425Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080318Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080227Sergey Koren A src/utgcns/libcns/abAbacus.C jcvi20080213Eli Venter A src/utgcns/libcns/abAbacus.C jcvi20080204Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20080124Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20080101Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20071220Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20071217Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20071208Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20071108Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20071025Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20071003Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20071002Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070925Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070918Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070905Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070831Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070829Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070829Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070807Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070804Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070803Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070617Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070612Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070610Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070601Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070529Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070428Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070426Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070419Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070416Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070402Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070319Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070316Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070310Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20070305Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070304Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070226Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070225Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070218Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070214Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070212Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070208Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070204Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070203Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20070128Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20061212Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20061117Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061024Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061021Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061020Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061017Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061016Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061016Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20061015Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061013Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061010Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061009Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061008Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20061005Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061003Aaron Halpern A src/utgcns/libcns/abAbacus.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060926Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060924Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060922Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060918Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060913Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060909Aaron Halpern A src/utgcns/libcns/abAbacus.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060907Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060906Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060905Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060829Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060821Aaron Halpern A src/utgcns/libcns/abAbacus.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060804Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060612Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060602Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060601Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060524Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060521Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060520Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060519Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060503Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060406Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060328Aaron Halpern A src/utgcns/libcns/abAbacus.C jcvi20060324Eli Venter A src/utgcns/libcns/abAbacus.C jcvi20060323Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060322Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060317Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060316Eli Venter A src/utgcns/libcns/abAbacus.C jcvi20060308Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20060302Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060214Eli Venter A src/utgcns/libcns/abAbacus.C jcvi20060130Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20060112Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20051216Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20051215Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20051213Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20051212Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20051212Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20051127Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20051120Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20051001Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus.C jcvi20050926Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20050920Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20050918Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050917Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050914Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050912Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20050911Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050827Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050819Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20050819Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050818Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050812Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050810Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050808Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050805Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050727Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050708Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20050630Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050629Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050623Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050621Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050616Brian P. Walenz A src/utgcns/libcns/abAbacus.C jcvi20050614Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050612Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050523Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050509Gennady Denisov A src/utgcns/libcns/abAbacus.C jcvi20050502Eli Venter A src/utgcns/libcns/abAbacus.C jcvi20050330Eli Venter A src/utgcns/libcns/abAbacus.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus.C tigr20040923Michael Schatz D src/utgcns/libcns/abAbacus.C src/AS_CNS/MultiAlignment_CNS.C D src/utgcns/libcns/abAbacus.C src/AS_CNS/MultiAlignment_CNS.c D src/utgcns/libcns/abAbacus.C src/utgcns/libcns/MultiAlignment_CNS.C A src/utgcns/libcns/abColumn.C bnbi20150121Brian P. Walenz A src/utgcns/libcns/abColumn.C bnbi20150113Brian P. Walenz A src/utgcns/libcns/abColumn.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/abColumn.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/abColumn.C bnbi20141230Brian P. Walenz A src/utgcns/libcns/abColumn.C bnbi20141117Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20091010Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090914Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090730Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090522Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090519Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090516Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090515Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090514Sergey Koren A src/utgcns/libcns/abColumn.C jcvi20090421Sergey Koren A src/utgcns/libcns/abColumn.C jcvi20090417Sergey Koren A src/utgcns/libcns/abColumn.C jcvi20090415Sergey Koren A src/utgcns/libcns/abColumn.C jcvi20090223Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090126Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090116Sergey Koren A src/utgcns/libcns/abColumn.C jcvi20090113Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090108Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090107Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090104Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20090102Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081219Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081205Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081113Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081112Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081107Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081105Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081008Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081007Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080925Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080731Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080627Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080606Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20080425Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080318Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080227Sergey Koren A src/utgcns/libcns/abColumn.C jcvi20080213Eli Venter A src/utgcns/libcns/abColumn.C jcvi20080204Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20080124Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20080101Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20071220Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20071217Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20071208Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20071108Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20071025Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20071003Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20071002Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070925Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070918Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070905Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070831Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070829Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070829Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070807Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070804Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070803Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070617Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070612Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070610Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070601Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070529Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070428Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070426Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070419Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070416Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070402Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070319Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070316Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070310Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20070305Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070304Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070226Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070225Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070218Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070214Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070212Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070208Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070204Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070203Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20070128Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20061212Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20061117Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20061114Eli Venter A src/utgcns/libcns/abColumn.C jcvi20061114Eli Venter A src/utgcns/libcns/abColumn.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061024Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061021Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061020Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061017Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061016Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061016Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20061015Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061013Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061010Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061009Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061008Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20061005Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061003Aaron Halpern A src/utgcns/libcns/abColumn.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060926Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060924Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060922Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060918Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060913Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060909Aaron Halpern A src/utgcns/libcns/abColumn.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060907Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060906Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060905Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060829Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060821Aaron Halpern A src/utgcns/libcns/abColumn.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060804Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060612Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060602Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060601Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060524Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060521Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060520Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060519Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060503Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060406Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060328Aaron Halpern A src/utgcns/libcns/abColumn.C jcvi20060324Eli Venter A src/utgcns/libcns/abColumn.C jcvi20060323Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060322Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060317Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060316Eli Venter A src/utgcns/libcns/abColumn.C jcvi20060308Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20060302Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060214Eli Venter A src/utgcns/libcns/abColumn.C jcvi20060130Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20060112Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20051216Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20051215Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20051213Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20051212Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20051212Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20051127Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20051120Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20051001Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abColumn.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abColumn.C jcvi20050926Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20050920Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20050918Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050917Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050914Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050912Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20050911Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050827Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050819Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20050819Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050818Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050812Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050810Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050808Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050805Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050727Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050708Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20050630Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050629Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050623Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050621Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050616Brian P. Walenz A src/utgcns/libcns/abColumn.C jcvi20050614Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050612Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050523Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050509Gennady Denisov A src/utgcns/libcns/abColumn.C jcvi20050502Eli Venter A src/utgcns/libcns/abColumn.C jcvi20050330Eli Venter A src/utgcns/libcns/abColumn.C tigr20050322Jason Miller A src/utgcns/libcns/abColumn.C tigr20050322Jason Miller A src/utgcns/libcns/abColumn.C tigr20040923Michael Schatz D src/utgcns/libcns/abColumn.C src/AS_CNS/MultiAlignment_CNS.C D src/utgcns/libcns/abColumn.C src/AS_CNS/MultiAlignment_CNS.c D src/utgcns/libcns/abColumn.C src/utgcns/libcns/MultiAlignment_CNS.C A src/utgcns/libcns/abBead.H bnbi20150127Brian P. Walenz A src/utgcns/libcns/abBead.H bnbi20150121Brian P. Walenz A src/utgcns/libcns/abBead.H bnbi20150108Brian P. Walenz A src/utgcns/libcns/abBead.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abBead.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abBead.H bnbi20141230Brian P. Walenz A src/utgcns/libcns/abBead.H bnbi20141223Brian P. Walenz A src/utgcns/libcns/abBead.H bnbi20141117Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20130107Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111215Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111208Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111206Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111129Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111117Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20111115Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20110921Jason Miller A src/utgcns/libcns/abBead.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20091026Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090925Sergey Koren A src/utgcns/libcns/abBead.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090914Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090907Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090804Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090710Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090622Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090421Sergey Koren A src/utgcns/libcns/abBead.H jcvi20090417Sergey Koren A src/utgcns/libcns/abBead.H jcvi20090415Sergey Koren A src/utgcns/libcns/abBead.H jcvi20090108Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20090105Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20081230Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20081218Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20081211Sergey Koren A src/utgcns/libcns/abBead.H jcvi20081107Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20080924Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20080627Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20080609Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20080425Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20080213Eli Venter A src/utgcns/libcns/abBead.H jcvi20080128Sergey Koren A src/utgcns/libcns/abBead.H jcvi20071108Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20071025Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20070925Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20070918Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20070905Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20070514Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20070316Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20070225Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20070214Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20070212Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20070208Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20061005Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20061002Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20060923Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20060918Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20060906Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20060612Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20060213Eli Venter A src/utgcns/libcns/abBead.H jcvi20051215Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20051127Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20051120Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20050829Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20050818Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20050805Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20050708Brian P. Walenz A src/utgcns/libcns/abBead.H jcvi20050629Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20050614Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20050612Gennady Denisov A src/utgcns/libcns/abBead.H jcvi20050523Gennady Denisov D src/utgcns/libcns/abBead.H src/AS_CNS/MultiAlignment_CNS.h D src/utgcns/libcns/abBead.H src/AS_CNS/MultiAlignment_CNS_private.H D src/utgcns/libcns/abBead.H src/AS_CNS/MultiAlignment_CNS_private.h D src/utgcns/libcns/abBead.H src/utgcns/libcns/MultiAlignment_CNS_private.H A src/utgcns/libcns/abAbacus-baseCall.C bnbi20150728Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20150123Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20150121Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20150121Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20150110Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20141230Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C bnbi20141117Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111221Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111208Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20100819Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20091010Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090914Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090730Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090709Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090522Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090519Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090516Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090515Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090514Sergey Koren A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090421Sergey Koren A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090417Sergey Koren A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090415Sergey Koren A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090223Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090126Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090116Sergey Koren A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090113Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090108Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090107Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090104Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20090102Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081219Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081205Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081113Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081112Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081107Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081105Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081008Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081007Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080925Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080731Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080627Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080606Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080425Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080318Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080227Sergey Koren A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080213Eli Venter A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080204Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080124Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20080101Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071220Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071217Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071208Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071108Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071025Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071003Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20071002Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070925Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070918Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070905Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070831Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070829Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070829Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070807Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070804Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070803Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070617Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070612Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070610Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070601Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070529Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070428Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070426Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070419Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070416Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070402Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070319Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070316Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070310Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070305Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070304Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070226Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070225Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070218Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070214Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070212Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070208Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070204Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070203Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20070128Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061212Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061117Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061024Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061021Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061020Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061017Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061016Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061016Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061015Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061013Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061010Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061009Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061008Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061005Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061003Aaron Halpern A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060926Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060924Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060922Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060918Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060913Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060909Aaron Halpern A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060907Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060906Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060905Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060829Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060821Aaron Halpern A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060804Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060612Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060602Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060601Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060524Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060521Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060520Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060519Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060503Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060406Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060328Aaron Halpern A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060324Eli Venter A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060323Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060322Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060317Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060316Eli Venter A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060308Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060302Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060214Eli Venter A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060130Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20060112Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20051216Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20051215Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20051213Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20051212Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20051212Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20051127Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20051120Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20051001Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050926Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050920Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050918Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050917Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050914Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050912Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050911Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050827Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050819Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050819Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050818Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050812Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050810Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050808Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050805Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050727Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050708Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050630Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050629Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050623Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050621Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050616Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050614Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050612Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050523Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050509Gennady Denisov A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050502Eli Venter A src/utgcns/libcns/abAbacus-baseCall.C jcvi20050330Eli Venter A src/utgcns/libcns/abAbacus-baseCall.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-baseCall.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-baseCall.C tigr20040923Michael Schatz D src/utgcns/libcns/abAbacus-baseCall.C src/AS_CNS/BaseCall.C D src/utgcns/libcns/abAbacus-baseCall.C src/AS_CNS/BaseCall.c D src/utgcns/libcns/abAbacus-baseCall.C src/AS_CNS/MultiAlignment_CNS.c D src/utgcns/libcns/abAbacus-baseCall.C src/utgcns/libcns/BaseCall.C A src/utgcns/libcns/abAbacus-refreshMultiAlign.C nihh20151014Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C bnbi20150306Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C bnbi20150123Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C bnbi20150110Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C bnbi20150109Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C bnbi20141230Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C bnbi20141117Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111208Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20091010Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090925Sergey Koren A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090914Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090730Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090522Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090519Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090516Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090515Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090514Sergey Koren A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090421Sergey Koren A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090417Sergey Koren A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090415Sergey Koren A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090223Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090126Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090116Sergey Koren A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090113Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090108Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090107Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090104Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20090102Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081219Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081205Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081113Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081112Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081107Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081105Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081008Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081007Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080925Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080731Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080627Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080606Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080425Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080318Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080227Sergey Koren A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080213Eli Venter A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080204Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080124Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20080101Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071220Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071217Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071208Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071108Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071025Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071003Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20071002Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070925Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070918Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070905Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070831Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070829Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070829Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070807Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070804Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070803Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070617Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070612Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070610Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070601Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070529Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070428Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070426Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070419Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070416Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070402Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070319Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070316Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070310Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070305Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070304Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070226Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070225Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070218Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070214Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070212Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070208Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070204Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070203Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20070128Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061212Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061117Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061024Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061021Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061020Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061017Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061016Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061016Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061015Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061013Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061010Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061009Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061008Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061005Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061003Aaron Halpern A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060926Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060924Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060922Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060918Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060913Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060909Aaron Halpern A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060907Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060906Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060905Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060829Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060821Aaron Halpern A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060804Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060612Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060602Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060601Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060524Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060521Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060520Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060519Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060503Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060406Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060328Aaron Halpern A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060324Eli Venter A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060323Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060322Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060317Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060316Eli Venter A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060308Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060302Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060214Eli Venter A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060130Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20060112Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20051216Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20051215Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20051213Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20051212Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20051212Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20051127Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20051120Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20051001Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050926Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050920Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050918Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050917Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050914Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050912Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050911Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050827Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050819Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050819Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050818Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050812Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050810Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050808Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050805Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050727Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050708Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050630Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050629Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050623Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050621Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050616Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050614Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050612Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050523Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050509Gennady Denisov A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050502Eli Venter A src/utgcns/libcns/abAbacus-refreshMultiAlign.C jcvi20050330Eli Venter A src/utgcns/libcns/abAbacus-refreshMultiAlign.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-refreshMultiAlign.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-refreshMultiAlign.C tigr20040923Michael Schatz D src/utgcns/libcns/abAbacus-refreshMultiAlign.C src/AS_CNS/MultiAlignment_CNS.c D src/utgcns/libcns/abAbacus-refreshMultiAlign.C src/AS_CNS/RefreshMANode.C D src/utgcns/libcns/abAbacus-refreshMultiAlign.C src/AS_CNS/RefreshMANode.c D src/utgcns/libcns/abAbacus-refreshMultiAlign.C src/utgcns/libcns/RefreshMANode.C A src/utgcns/libcns/abAbacus.H nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151014Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151010Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151009Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150723Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150708Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150528Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150204Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150204Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150127Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150123Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150121Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150121Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150113Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150112Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150110Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150109Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150108Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150108Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20141230Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20141223Brian P. Walenz A src/utgcns/libcns/abAbacus.H bnbi20141117Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20130107Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111215Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111208Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111206Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111129Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111117Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20111115Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20110921Jason Miller A src/utgcns/libcns/abAbacus.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20091026Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090925Sergey Koren A src/utgcns/libcns/abAbacus.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090914Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090907Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090804Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090710Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090421Sergey Koren A src/utgcns/libcns/abAbacus.H jcvi20090417Sergey Koren A src/utgcns/libcns/abAbacus.H jcvi20090415Sergey Koren A src/utgcns/libcns/abAbacus.H jcvi20090108Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20081211Sergey Koren A src/utgcns/libcns/abAbacus.H jcvi20081107Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20080627Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20080609Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20080425Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20080213Eli Venter A src/utgcns/libcns/abAbacus.H jcvi20080128Sergey Koren A src/utgcns/libcns/abAbacus.H jcvi20071108Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20071025Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20070925Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20070918Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20070905Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20070514Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20070316Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20070225Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20070214Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20070212Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20070208Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20061005Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20060923Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20060918Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20060906Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20060612Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20060213Eli Venter A src/utgcns/libcns/abAbacus.H jcvi20051215Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20051127Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20051120Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20050829Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20050818Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20050805Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20050708Brian P. Walenz A src/utgcns/libcns/abAbacus.H jcvi20050629Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20050614Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20050612Gennady Denisov A src/utgcns/libcns/abAbacus.H jcvi20050523Gennady Denisov D src/utgcns/libcns/abAbacus.H src/AS_CNS/MultiAlignment_CNS.h D src/utgcns/libcns/abAbacus.H src/AS_CNS/MultiAlignment_CNS_private.H D src/utgcns/libcns/abAbacus.H src/AS_CNS/MultiAlignment_CNS_private.h D src/utgcns/libcns/abAbacus.H src/utgcns/libcns/MultiAlignment_CNS_private.H A src/utgcns/libcns/abMultiAlign.C bnbi20150811Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150730Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150708Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150702Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150507Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150306Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150128Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150123Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150113Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150110Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20141230Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C bnbi20141117Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20091010Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090914Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090730Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090522Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090519Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090516Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090515Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090514Sergey Koren A src/utgcns/libcns/abMultiAlign.C jcvi20090421Sergey Koren A src/utgcns/libcns/abMultiAlign.C jcvi20090417Sergey Koren A src/utgcns/libcns/abMultiAlign.C jcvi20090415Sergey Koren A src/utgcns/libcns/abMultiAlign.C jcvi20090223Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090126Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090116Sergey Koren A src/utgcns/libcns/abMultiAlign.C jcvi20090113Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090108Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090107Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090104Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20090102Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081219Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081205Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081113Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081112Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081107Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081105Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081008Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081007Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080925Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080731Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080627Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080606Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20080425Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080318Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080227Sergey Koren A src/utgcns/libcns/abMultiAlign.C jcvi20080213Eli Venter A src/utgcns/libcns/abMultiAlign.C jcvi20080204Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20080124Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20080101Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20071220Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20071217Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20071208Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20071108Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20071025Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20071003Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20071002Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070925Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070918Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070905Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070831Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070829Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070829Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070807Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070804Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070803Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070617Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070612Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070610Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070601Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070529Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070428Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070426Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070419Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070416Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070402Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070319Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070316Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070310Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20070305Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070304Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070226Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070225Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070218Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070214Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070212Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070208Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070204Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070203Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20070128Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20061212Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20061117Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20061114Eli Venter A src/utgcns/libcns/abMultiAlign.C jcvi20061114Eli Venter A src/utgcns/libcns/abMultiAlign.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061024Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061021Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061020Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061017Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061016Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061016Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20061015Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061013Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061010Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061009Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061008Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20061005Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061003Aaron Halpern A src/utgcns/libcns/abMultiAlign.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060926Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060924Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060922Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060918Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060913Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060909Aaron Halpern A src/utgcns/libcns/abMultiAlign.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060907Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060906Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060905Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060829Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060821Aaron Halpern A src/utgcns/libcns/abMultiAlign.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060804Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060612Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060602Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060601Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060524Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060521Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060520Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060519Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060503Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060406Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060328Aaron Halpern A src/utgcns/libcns/abMultiAlign.C jcvi20060324Eli Venter A src/utgcns/libcns/abMultiAlign.C jcvi20060323Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060322Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060317Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060316Eli Venter A src/utgcns/libcns/abMultiAlign.C jcvi20060308Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20060302Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060214Eli Venter A src/utgcns/libcns/abMultiAlign.C jcvi20060130Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20060112Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20051216Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20051215Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20051213Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20051212Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20051212Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20051127Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20051120Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20051001Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abMultiAlign.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abMultiAlign.C jcvi20050926Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20050920Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20050918Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050917Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050914Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050912Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20050911Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050827Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050819Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20050819Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050818Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050812Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050810Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050808Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050805Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050727Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050708Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20050630Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050629Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050623Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050621Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050616Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C jcvi20050614Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050612Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050523Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050509Gennady Denisov A src/utgcns/libcns/abMultiAlign.C jcvi20050502Eli Venter A src/utgcns/libcns/abMultiAlign.C jcvi20050330Eli Venter A src/utgcns/libcns/abMultiAlign.C tigr20050322Jason Miller A src/utgcns/libcns/abMultiAlign.C tigr20050322Jason Miller A src/utgcns/libcns/abMultiAlign.C tigr20040923Michael Schatz D src/utgcns/libcns/abMultiAlign.C src/AS_CNS/MultiAlignment_CNS.C D src/utgcns/libcns/abMultiAlign.C src/AS_CNS/MultiAlignment_CNS.c D src/utgcns/libcns/abMultiAlign.C src/utgcns/libcns/MultiAlignment_CNS.C A src/utgcns/libcns/abAbacus-mergeRefine.C nihh20151014Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C bnbi20150306Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C bnbi20150123Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C bnbi20150110Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C bnbi20150108Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C bnbi20141230Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C bnbi20141117Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20091010Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090914Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090730Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090522Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090519Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090516Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090515Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090514Sergey Koren A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090421Sergey Koren A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090417Sergey Koren A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090415Sergey Koren A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090223Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090126Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090116Sergey Koren A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090113Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090108Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090107Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090104Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20090102Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081219Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081205Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081113Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081112Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081107Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081105Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081008Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081007Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080925Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080731Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080627Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080606Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080425Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080318Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080227Sergey Koren A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080213Eli Venter A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080204Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080124Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20080101Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071220Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071217Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071208Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071108Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071025Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071003Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20071002Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070925Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070918Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070905Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070831Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070829Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070829Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070807Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070804Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070803Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070617Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070612Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070610Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070601Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070529Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070428Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070426Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070419Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070416Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070402Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070319Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070316Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070310Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070305Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070304Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070226Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070225Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070218Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070214Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070212Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070208Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070204Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070203Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20070128Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061212Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061117Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061024Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061021Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061020Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061017Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061016Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061016Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061015Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061013Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061010Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061009Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061008Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061005Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061003Aaron Halpern A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060926Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060924Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060922Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060918Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060913Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060909Aaron Halpern A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060907Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060906Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060905Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060829Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060821Aaron Halpern A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060804Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060612Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060602Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060601Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060524Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060521Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060520Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060519Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060503Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060406Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060328Aaron Halpern A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060324Eli Venter A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060323Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060322Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060317Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060316Eli Venter A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060308Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060302Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060214Eli Venter A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060130Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20060112Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20051216Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20051215Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20051213Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20051212Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20051212Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20051127Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20051120Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20051001Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050926Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050920Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050918Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050917Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050914Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050912Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050911Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050827Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050819Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050819Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050818Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050812Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050810Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050808Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050805Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050727Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050708Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050630Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050629Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050623Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050621Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050616Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050614Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050612Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050523Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050509Gennady Denisov A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050502Eli Venter A src/utgcns/libcns/abAbacus-mergeRefine.C jcvi20050330Eli Venter A src/utgcns/libcns/abAbacus-mergeRefine.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-mergeRefine.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-mergeRefine.C tigr20040923Michael Schatz D src/utgcns/libcns/abAbacus-mergeRefine.C src/AS_CNS/MergeRefine.C D src/utgcns/libcns/abAbacus-mergeRefine.C src/AS_CNS/MergeRefine.c D src/utgcns/libcns/abAbacus-mergeRefine.C src/AS_CNS/MultiAlignment_CNS.c D src/utgcns/libcns/abAbacus-mergeRefine.C src/utgcns/libcns/MergeRefine.C A src/utgcns/libcns/abSequence.H bnbi20150701Brian P. Walenz A src/utgcns/libcns/abSequence.H bnbi20150127Brian P. Walenz A src/utgcns/libcns/abSequence.H bnbi20150121Brian P. Walenz A src/utgcns/libcns/abSequence.H bnbi20150112Brian P. Walenz A src/utgcns/libcns/abSequence.H bnbi20150110Brian P. Walenz A src/utgcns/libcns/abSequence.H bnbi20150108Brian P. Walenz A src/utgcns/libcns/abSequence.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abSequence.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abSequence.H bnbi20141230Brian P. Walenz A src/utgcns/libcns/abSequence.H bnbi20141223Brian P. Walenz A src/utgcns/libcns/abSequence.H bnbi20141117Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20130107Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111215Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111208Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111206Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111129Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111117Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20111115Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20110921Jason Miller A src/utgcns/libcns/abSequence.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20091026Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090925Sergey Koren A src/utgcns/libcns/abSequence.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090914Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090907Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090804Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090710Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090622Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090421Sergey Koren A src/utgcns/libcns/abSequence.H jcvi20090417Sergey Koren A src/utgcns/libcns/abSequence.H jcvi20090415Sergey Koren A src/utgcns/libcns/abSequence.H jcvi20090108Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20090105Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20081230Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20081218Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20081211Sergey Koren A src/utgcns/libcns/abSequence.H jcvi20081107Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20080924Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20080627Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20080609Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20080425Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20080213Eli Venter A src/utgcns/libcns/abSequence.H jcvi20080128Sergey Koren A src/utgcns/libcns/abSequence.H jcvi20071108Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20071025Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20070925Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20070918Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20070905Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20070514Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20070316Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20070225Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20070214Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20070212Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20070208Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20061005Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20061002Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20060923Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20060918Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20060906Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20060612Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20060213Eli Venter A src/utgcns/libcns/abSequence.H jcvi20051215Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20051127Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20051120Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20050829Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20050818Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20050805Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20050708Brian P. Walenz A src/utgcns/libcns/abSequence.H jcvi20050629Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20050614Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20050612Gennady Denisov A src/utgcns/libcns/abSequence.H jcvi20050523Gennady Denisov D src/utgcns/libcns/abSequence.H src/AS_CNS/MultiAlignment_CNS.h D src/utgcns/libcns/abSequence.H src/AS_CNS/MultiAlignment_CNS_private.H D src/utgcns/libcns/abSequence.H src/AS_CNS/MultiAlignment_CNS_private.h D src/utgcns/libcns/abSequence.H src/utgcns/libcns/MultiAlignment_CNS_private.H A src/utgcns/libcns/unitigConsensus.C nihh20151113Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151020Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151013Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151009Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150811Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150810Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150805Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150730Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150728Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150726Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150723Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150722Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150720Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150720Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150717Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150715Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150715Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150715Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150715Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150713Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150708Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150708Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150623Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150528Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150306Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150303Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150204Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150128Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150123Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150121Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150113Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150112Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150110Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20141230Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20141117Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20131004Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130918Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130918Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130806Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130730Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130703Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20120726Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111229Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111218Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111214Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111129Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111123Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111123Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111114Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111111Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C bnbi20111027Sergey Koren A src/utgcns/libcns/unitigConsensus.C jcvi20110728Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20110224Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20110104Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20100812Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20100423Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20100318Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20100316Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20100302Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20091209Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20091130Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20091104Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20091010Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090914Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090811Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090807Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090731Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090730Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090726Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090715Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090715Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090713Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090707Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090707Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090629Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090624Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090624Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090624Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090605Sergey Koren A src/utgcns/libcns/unitigConsensus.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090522Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090519Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090516Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090515Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090514Sergey Koren A src/utgcns/libcns/unitigConsensus.C jcvi20090421Sergey Koren A src/utgcns/libcns/unitigConsensus.C jcvi20090417Sergey Koren A src/utgcns/libcns/unitigConsensus.C jcvi20090415Sergey Koren A src/utgcns/libcns/unitigConsensus.C jcvi20090223Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090126Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090116Sergey Koren A src/utgcns/libcns/unitigConsensus.C jcvi20090113Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090108Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090107Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090104Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20090102Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081219Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081205Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081113Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081112Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081107Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081105Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081008Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081007Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080925Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080731Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080627Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080606Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20080425Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080318Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080227Sergey Koren A src/utgcns/libcns/unitigConsensus.C jcvi20080213Eli Venter A src/utgcns/libcns/unitigConsensus.C jcvi20080204Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20080124Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20080101Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20071220Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20071217Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20071208Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20071108Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20071025Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20071017Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20071017Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20071003Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20071002Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070925Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070918Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070905Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070831Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070830Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070830Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070829Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070829Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070807Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070804Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070803Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070617Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070612Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070610Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070601Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070529Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070428Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070426Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070419Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070416Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070402Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070319Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070316Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070310Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20070305Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070304Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070226Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070225Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070218Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070214Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070212Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070208Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070204Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070203Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20070128Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20061212Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20061117Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20061114Eli Venter A src/utgcns/libcns/unitigConsensus.C jcvi20061114Eli Venter A src/utgcns/libcns/unitigConsensus.C jcvi20061029Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061029Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061025Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061025Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061024Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061023Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061023Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061021Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061020Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061017Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20061017Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061017Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061016Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061016Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20061015Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061013Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061010Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061009Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061008Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20061005Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061003Aaron Halpern A src/utgcns/libcns/unitigConsensus.C jcvi20061002Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20061002Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060926Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060924Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060922Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060918Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060913Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060909Aaron Halpern A src/utgcns/libcns/unitigConsensus.C jcvi20060908Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060908Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060907Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060906Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060905Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060829Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060825Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060825Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060821Aaron Halpern A src/utgcns/libcns/unitigConsensus.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060818Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060818Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060804Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060612Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060602Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060601Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060524Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060521Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060520Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060519Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060503Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060406Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060328Aaron Halpern A src/utgcns/libcns/unitigConsensus.C jcvi20060324Eli Venter A src/utgcns/libcns/unitigConsensus.C jcvi20060323Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060322Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060317Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060316Eli Venter A src/utgcns/libcns/unitigConsensus.C jcvi20060308Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20060302Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060214Eli Venter A src/utgcns/libcns/unitigConsensus.C jcvi20060130Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20060112Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20051216Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20051215Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20051213Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20051212Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20051212Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20051127Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20051120Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20051001Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050930Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050930Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050929Aaron Halpern A src/utgcns/libcns/unitigConsensus.C jcvi20050929Aaron Halpern A src/utgcns/libcns/unitigConsensus.C jcvi20050926Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20050920Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20050918Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050917Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050914Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050912Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20050911Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050827Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050819Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20050819Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050818Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050812Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050810Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050808Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050805Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050801Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050801Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050801Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050801Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050727Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050708Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20050630Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050629Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050623Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050621Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050616Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C jcvi20050614Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050612Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050523Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050509Gennady Denisov A src/utgcns/libcns/unitigConsensus.C jcvi20050502Eli Venter A src/utgcns/libcns/unitigConsensus.C jcvi20050330Eli Venter A src/utgcns/libcns/unitigConsensus.C tigr20050322Jason Miller A src/utgcns/libcns/unitigConsensus.C tigr20050322Jason Miller A src/utgcns/libcns/unitigConsensus.C tigr20040923Michael Schatz D src/utgcns/libcns/unitigConsensus.C src/AS_CNS/MultiAlignUnitig.C D src/utgcns/libcns/unitigConsensus.C src/AS_CNS/MultiAlignUnitig.c D src/utgcns/libcns/unitigConsensus.C src/AS_CNS/MultiAlignment_CNS.c D src/utgcns/libcns/unitigConsensus.C src/utgcns/libcns/MultiAlignUnitig.C A src/utgcns/libcns/abIterators.H bnbi20150131Brian P. Walenz A src/utgcns/libcns/abIterators.H bnbi20150113Brian P. Walenz A src/utgcns/libcns/abIterators.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abIterators.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abIterators.H bnbi20141230Brian P. Walenz A src/utgcns/libcns/abIterators.H bnbi20141223Brian P. Walenz A src/utgcns/libcns/abIterators.H bnbi20141117Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20130107Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111215Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111208Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111206Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111129Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111117Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20111115Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20110921Jason Miller A src/utgcns/libcns/abIterators.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20091026Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090925Sergey Koren A src/utgcns/libcns/abIterators.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090914Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090907Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090804Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090710Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090622Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090421Sergey Koren A src/utgcns/libcns/abIterators.H jcvi20090417Sergey Koren A src/utgcns/libcns/abIterators.H jcvi20090415Sergey Koren A src/utgcns/libcns/abIterators.H jcvi20090108Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20090105Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20081230Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20081218Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20081211Sergey Koren A src/utgcns/libcns/abIterators.H jcvi20081107Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20080924Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20080627Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20080609Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20080425Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20080213Eli Venter A src/utgcns/libcns/abIterators.H jcvi20080128Sergey Koren A src/utgcns/libcns/abIterators.H jcvi20071108Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20071025Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20070925Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20070918Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20070905Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20070514Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20070316Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20070225Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20070214Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20070212Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20070208Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20061005Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20061002Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20060923Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20060918Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20060906Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20060612Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20060213Eli Venter A src/utgcns/libcns/abIterators.H jcvi20051215Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20051127Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20051120Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20050829Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20050818Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20050805Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20050708Brian P. Walenz A src/utgcns/libcns/abIterators.H jcvi20050629Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20050614Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20050612Gennady Denisov A src/utgcns/libcns/abIterators.H jcvi20050523Gennady Denisov D src/utgcns/libcns/abIterators.H src/AS_CNS/MultiAlignment_CNS.h D src/utgcns/libcns/abIterators.H src/AS_CNS/MultiAlignment_CNS_private.H D src/utgcns/libcns/abIterators.H src/AS_CNS/MultiAlignment_CNS_private.h D src/utgcns/libcns/abIterators.H src/utgcns/libcns/MultiAlignment_CNS_private.H A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151113Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151029Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151014Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151013Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150728Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150726Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150715Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150708Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150317Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150306Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150127Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150123Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150121Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150110Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150108Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20150108Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20141230Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C bnbi20141117Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20091010Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090914Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090730Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090713Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090629Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090624Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090522Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090519Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090516Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090515Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090514Sergey Koren A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090421Sergey Koren A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090417Sergey Koren A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090415Sergey Koren A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090223Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090126Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090116Sergey Koren A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090113Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090108Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090107Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090104Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20090102Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081219Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081205Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081113Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081112Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081107Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081105Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081008Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081007Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080925Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080731Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080627Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080606Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080425Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080318Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080227Sergey Koren A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080213Eli Venter A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080204Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080124Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20080101Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071220Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071217Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071208Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071108Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071025Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071003Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20071002Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070925Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070918Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070905Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070831Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070829Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070829Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070807Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070804Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070803Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070617Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070612Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070610Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070601Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070529Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070428Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070426Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070419Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070416Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070402Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070319Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070316Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070310Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070305Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070304Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070226Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070225Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070218Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070214Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070212Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070208Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070204Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070203Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20070128Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061212Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061117Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061024Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061021Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061020Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061017Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061016Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061016Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061015Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061013Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061010Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061009Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061008Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061005Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061003Aaron Halpern A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060926Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060924Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060922Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060918Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060913Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060909Aaron Halpern A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060907Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060906Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060905Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060829Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060821Aaron Halpern A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060804Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060612Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060602Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060601Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060524Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060521Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060520Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060519Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060503Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060406Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060328Aaron Halpern A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060324Eli Venter A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060323Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060322Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060317Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060316Eli Venter A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060308Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060302Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060214Eli Venter A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060130Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20060112Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20051216Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20051215Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20051213Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20051212Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20051212Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20051127Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20051120Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20051001Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050926Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050920Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050918Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050917Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050914Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050912Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050911Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050827Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050819Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050819Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050818Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050812Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050810Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050808Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050805Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050727Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050708Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050630Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050629Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050623Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050621Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050616Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050614Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050612Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050523Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050509Gennady Denisov A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050502Eli Venter A src/utgcns/libcns/abAbacus-applyAlignment.C jcvi20050330Eli Venter A src/utgcns/libcns/abAbacus-applyAlignment.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-applyAlignment.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-applyAlignment.C tigr20040923Michael Schatz D src/utgcns/libcns/abAbacus-applyAlignment.C src/AS_CNS/ApplyAlignment.C D src/utgcns/libcns/abAbacus-applyAlignment.C src/AS_CNS/ApplyAlignment.c D src/utgcns/libcns/abAbacus-applyAlignment.C src/AS_CNS/MultiAlignment_CNS.c D src/utgcns/libcns/abAbacus-applyAlignment.C src/utgcns/libcns/ApplyAlignment.C A src/utgcns/libcns/abAbacus-addRead.C nihh20151009Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150811Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150708Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150603Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150528Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150317Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150317Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150131Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150121Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150112Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150110Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150108Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150108Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20150107Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20141230Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C bnbi20141117Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20130801Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20130617Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20111215Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20111212Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20111209Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20111207Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20111206Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20111204Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20111117Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20111115Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20111110Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20110102Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20100923Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20091026Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20091010Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20091005Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090929Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090924Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090914Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090804Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090730Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090710Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090622Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090610Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090529Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090527Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090522Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090520Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090519Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090516Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090515Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090514Sergey Koren A src/utgcns/libcns/abAbacus-addRead.C jcvi20090421Sergey Koren A src/utgcns/libcns/abAbacus-addRead.C jcvi20090417Sergey Koren A src/utgcns/libcns/abAbacus-addRead.C jcvi20090415Sergey Koren A src/utgcns/libcns/abAbacus-addRead.C jcvi20090223Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090126Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090116Sergey Koren A src/utgcns/libcns/abAbacus-addRead.C jcvi20090113Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090108Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090107Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090105Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090104Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20090102Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081231Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081230Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081229Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081219Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081218Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081205Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081113Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081112Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081107Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081105Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081029Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081011Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081008Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081007Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20081001Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080925Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080924Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080811Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080731Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080627Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080606Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20080425Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080404Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080318Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080227Sergey Koren A src/utgcns/libcns/abAbacus-addRead.C jcvi20080213Eli Venter A src/utgcns/libcns/abAbacus-addRead.C jcvi20080204Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20080124Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20080101Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20071220Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20071218Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20071217Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20071208Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20071108Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20071025Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20071017Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20071003Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20071002Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070925Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070918Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070905Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070902Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070901Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070831Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070830Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070829Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070829Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070807Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070804Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070803Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070617Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070612Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070610Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070601Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070601Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070529Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070518Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070428Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070426Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070419Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070416Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070402Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070319Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070316Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070310Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20070305Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070304Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070226Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070225Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070220Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070218Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070214Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070212Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070208Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070204Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070203Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070129Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20070128Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20061212Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20061117Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-addRead.C jcvi20061114Eli Venter A src/utgcns/libcns/abAbacus-addRead.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061029Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061025Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061024Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061023Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061021Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061020Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061017Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061017Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061016Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061016Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20061015Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061013Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061010Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061009Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061008Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20061005Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061003Aaron Halpern A src/utgcns/libcns/abAbacus-addRead.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20061002Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060926Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060924Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060922Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060918Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060913Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060909Aaron Halpern A src/utgcns/libcns/abAbacus-addRead.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060908Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060907Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060906Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060905Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060829Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060825Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060821Aaron Halpern A src/utgcns/libcns/abAbacus-addRead.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060821Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060818Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060804Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060612Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060602Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060601Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060524Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060521Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060520Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060519Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060503Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060406Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060328Aaron Halpern A src/utgcns/libcns/abAbacus-addRead.C jcvi20060324Eli Venter A src/utgcns/libcns/abAbacus-addRead.C jcvi20060323Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060322Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060317Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060316Eli Venter A src/utgcns/libcns/abAbacus-addRead.C jcvi20060308Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20060302Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060214Eli Venter A src/utgcns/libcns/abAbacus-addRead.C jcvi20060130Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20060112Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20051216Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20051215Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20051213Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20051212Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20051212Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20051127Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20051120Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20051001Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050930Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-addRead.C jcvi20050929Aaron Halpern A src/utgcns/libcns/abAbacus-addRead.C jcvi20050926Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20050920Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20050918Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050917Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050914Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050912Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20050911Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050827Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050819Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20050819Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050818Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050812Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050810Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050808Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050805Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050801Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050727Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050708Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20050630Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050629Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050623Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050621Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050616Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C jcvi20050614Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050612Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050523Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050509Gennady Denisov A src/utgcns/libcns/abAbacus-addRead.C jcvi20050502Eli Venter A src/utgcns/libcns/abAbacus-addRead.C jcvi20050330Eli Venter A src/utgcns/libcns/abAbacus-addRead.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-addRead.C tigr20050322Jason Miller A src/utgcns/libcns/abAbacus-addRead.C tigr20040923Michael Schatz D src/utgcns/libcns/abAbacus-addRead.C src/AS_CNS/MultiAlignment_CNS.C D src/utgcns/libcns/abAbacus-addRead.C src/AS_CNS/MultiAlignment_CNS.c D src/utgcns/libcns/abAbacus-addRead.C src/utgcns/libcns/MultiAlignment_CNS.C D src/utgcns/libcns/abAbacus-addRead.C src/utgcns/libcns/abacus-addRead.C A src/utgcns/libcns/abColumn.H bnbi20150701Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20150127Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20150121Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20150121Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20150113Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20150112Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20150110Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20150109Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20150107Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20141230Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20141223Brian P. Walenz A src/utgcns/libcns/abColumn.H bnbi20141117Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20130801Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20130107Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111215Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111208Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111207Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111206Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111204Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111129Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111117Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20111115Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20110921Jason Miller A src/utgcns/libcns/abColumn.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20110102Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20091026Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20091005Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090925Sergey Koren A src/utgcns/libcns/abColumn.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090924Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090914Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090907Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090804Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090710Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090622Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090610Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090529Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090515Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090421Sergey Koren A src/utgcns/libcns/abColumn.H jcvi20090417Sergey Koren A src/utgcns/libcns/abColumn.H jcvi20090415Sergey Koren A src/utgcns/libcns/abColumn.H jcvi20090108Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20090105Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20081230Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20081218Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20081211Sergey Koren A src/utgcns/libcns/abColumn.H jcvi20081107Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20081029Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20081008Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20080925Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20080924Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20080627Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20080609Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20080425Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20080213Eli Venter A src/utgcns/libcns/abColumn.H jcvi20080128Sergey Koren A src/utgcns/libcns/abColumn.H jcvi20071108Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20071025Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20070925Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20070918Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20070905Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20070514Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20070316Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20070225Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20070214Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20070212Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20070208Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20061005Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20061002Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20060923Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20060918Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20060906Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20060612Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20060213Eli Venter A src/utgcns/libcns/abColumn.H jcvi20051215Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20051127Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20051120Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20050829Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20050818Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20050805Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20050801Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20050708Brian P. Walenz A src/utgcns/libcns/abColumn.H jcvi20050629Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20050614Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20050612Gennady Denisov A src/utgcns/libcns/abColumn.H jcvi20050523Gennady Denisov D src/utgcns/libcns/abColumn.H src/AS_CNS/MultiAlignment_CNS.h D src/utgcns/libcns/abColumn.H src/AS_CNS/MultiAlignment_CNS_private.H D src/utgcns/libcns/abColumn.H src/AS_CNS/MultiAlignment_CNS_private.h D src/utgcns/libcns/abColumn.H src/utgcns/libcns/MultiAlignment_CNS_private.H A src/utgcns/libNDalign/NDalgorithm-reverse.C nihh20151029Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-reverse.C nihh20151014Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-reverse.C nihh20151013Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-reverse.C bnbi20150805Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-reverse.C bnbi20150728Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-reverse.C bnbi20150726Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-reverse.C bnbi20150723Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-reverse.C bnbi20150722Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-reverse.C bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-reverse.C bnbi20150720Brian P. Walenz D src/utgcns/libNDalign/NDalgorithm-reverse.C src/utgcns/libNDalign/prefixEditDistance-reverse.C A src/utgcns/libNDalign/NDalgorithm.C bnbi20150805Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm.C bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm.C bnbi20150720Brian P. Walenz D src/utgcns/libNDalign/NDalgorithm.C src/utgcns/libNDalign/prefixEditDistance.C A src/utgcns/libNDalign/NDalign.C nihh20151113Brian P. Walenz A src/utgcns/libNDalign/NDalign.C nihh20151029Brian P. Walenz A src/utgcns/libNDalign/NDalign.C nihh20151019Brian P. Walenz A src/utgcns/libNDalign/NDalign.C nihh20151013Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150825Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150730Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150728Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150726Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150723Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150717Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150715Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150715Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150713Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150708Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150708Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150701Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150624Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150623Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150623Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150617Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150617Brian P. Walenz A src/utgcns/libNDalign/NDalign.C bnbi20150617Brian P. Walenz D src/utgcns/libNDalign/NDalign.C src/overlapInCore/overlapAlign.C A src/utgcns/libNDalign/NDalgorithm-allocateMoreSpace.C bnbi20150805Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-allocateMoreSpace.C bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-allocateMoreSpace.C bnbi20150720Brian P. Walenz D src/utgcns/libNDalign/NDalgorithm-allocateMoreSpace.C src/utgcns/libNDalign/prefixEditDistance-allocateMoreSpace.C A src/utgcns/libNDalign/NDalgorithm.H nihh20151029Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm.H nihh20151020Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm.H nihh20151019Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm.H bnbi20150805Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm.H bnbi20150726Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm.H bnbi20150722Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm.H bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm.H bnbi20150720Brian P. Walenz D src/utgcns/libNDalign/NDalgorithm.H src/utgcns/libNDalign/prefixEditDistance.H A src/utgcns/libNDalign/NDalign.H bnbi20150825Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150728Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150726Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150723Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150717Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150708Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150708Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150701Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150625Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150624Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150623Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150623Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150617Brian P. Walenz A src/utgcns/libNDalign/NDalign.H bnbi20150617Brian P. Walenz D src/utgcns/libNDalign/NDalign.H src/overlapInCore/overlapAlign.H A src/utgcns/libNDalign/NDalgorithm-forward.C nihh20151029Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-forward.C nihh20151013Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-forward.C bnbi20150805Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-forward.C bnbi20150726Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-forward.C bnbi20150723Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-forward.C bnbi20150722Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-forward.C bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-forward.C bnbi20150720Brian P. Walenz D src/utgcns/libNDalign/NDalgorithm-forward.C src/utgcns/libNDalign/prefixEditDistance-forward.C A src/utgcns/libNDalign/NDalgorithm-extend.C nihh20151013Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-extend.C bnbi20150728Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-extend.C bnbi20150726Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-extend.C bnbi20150722Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-extend.C bnbi20150720Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-extend.C bnbi20150720Brian P. Walenz D src/utgcns/libNDalign/NDalgorithm-extend.C src/utgcns/libNDalign/prefixEditDistance-extend.C A src/utgcns/libNDalign/Binomial_Bound.C nihh20151012Brian P. Walenz A src/utgcns/stashContains.C bnbi20150509Brian P. Walenz A src/utgcns/stashContains.C bnbi20150409Brian P. Walenz A src/utgcns/utgcnsfix.C bnbi20150306Brian P. Walenz A src/utgcns/utgcnsfix.C bnbi20141230Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20131004Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20130918Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20130917Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20130801Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20130801Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20130308Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20120726Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20111229Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20111214Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20111212Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20111207Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20111204Brian P. Walenz A src/utgcns/utgcnsfix.C jcvi20111129Brian P. Walenz D src/utgcns/utgcnsfix.C src/AS_CNS/utgcnsfix.C A src/pipelines/canu.pl nihh20151119Brian P. Walenz A src/pipelines/canu.pl nihh20151116Brian P. Walenz A src/pipelines/canu.pl nihh20151109Brian P. Walenz A src/pipelines/canu.pl nihh20151108Brian P. Walenz A src/pipelines/canu.pl nihh20151108Brian P. Walenz A src/pipelines/canu.pl nihh20151104Brian P. Walenz A src/pipelines/canu.pl nihh20151103Brian P. Walenz A src/pipelines/canu.pl bnbi20150826Brian P. Walenz A src/pipelines/canu.pl bnbi20150826Brian P. Walenz A src/pipelines/canu.pl bnbi20150826Brian P. Walenz A src/pipelines/canu.pl bnbi20150825Brian P. Walenz A src/pipelines/canu.pl bnbi20150814Brian P. Walenz A src/pipelines/canu.pl bnbi20150814Brian P. Walenz A src/pipelines/canu.pl bnbi20150616Brian P. Walenz A src/pipelines/canu.pl bnbi20150616Brian P. Walenz A src/pipelines/canu.pl bnbi20150603Brian P. Walenz A src/pipelines/canu.pl bnbi20150528Brian P. Walenz A src/pipelines/canu.pl bnbi20150526Brian P. Walenz A src/pipelines/canu.pl bnbi20150514Brian P. Walenz A src/pipelines/canu.pl bnbi20150507Brian P. Walenz A src/pipelines/canu.pl bnbi20150429Brian P. Walenz A src/pipelines/canu.pl bnbi20150424Brian P. Walenz A src/pipelines/canu.pl bnbi20150421Brian P. Walenz A src/pipelines/canu.pl bnbi20150415Brian P. Walenz A src/pipelines/canu.pl bnbi20150415Brian P. Walenz A src/pipelines/canu.pl bnbi20150410Brian P. Walenz A src/pipelines/canu.pl bnbi20150409Brian P. Walenz A src/pipelines/canu.pl bnbi20150409Brian P. Walenz A src/pipelines/canu.pl bnbi20150327Brian P. Walenz A src/pipelines/canu.pl bnbi20150327Brian P. Walenz A src/pipelines/canu.pl bnbi20150327Brian P. Walenz A src/pipelines/canu.pl bnbi20150325Brian P. Walenz A src/pipelines/canu.pl bnbi20150321Brian P. Walenz A src/pipelines/canu.pl bnbi20150317Brian P. Walenz A src/pipelines/canu.pl bnbi20150316Brian P. Walenz A src/pipelines/canu.pl bnbi20150310Brian P. Walenz A src/pipelines/canu.pl bnbi20150306Brian P. Walenz A src/pipelines/canu.pl bnbi20150306Brian P. Walenz A src/pipelines/canu.pl bnbi20150303Brian P. Walenz A src/pipelines/canu.pl bnbi20150227Brian P. Walenz D src/pipelines/canu.pl src/pipelines/ca3g.pl A src/pipelines/canu/OverlapInCore.pm nihh20151119Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20151109Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20151108Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20151104Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20151103Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20151027Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20151019Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150421Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150420Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150327Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150327Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150317Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150310Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150306Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150303Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm bnbi20150227Brian P. Walenz D src/pipelines/canu/OverlapInCore.pm src/pipelines/ca3g/OverlapInCore.pm A src/pipelines/canu/HTML.pm nihh20151118Brian P. Walenz A src/pipelines/canu/HTML.pm nihh20151109Brian P. Walenz A src/pipelines/canu/HTML.pm nihh20151109Brian P. Walenz A src/pipelines/canu/HTML.pm nihh20151108Brian P. Walenz A src/pipelines/canu/Output.pm nihh20151118Brian P. Walenz A src/pipelines/canu/Output.pm nihh20151109Brian P. Walenz A src/pipelines/canu/Output.pm nihh20151108Brian P. Walenz A src/pipelines/canu/Output.pm nihh20151105Brian P. Walenz A src/pipelines/canu/Output.pm nihh20151104Brian P. Walenz A src/pipelines/canu/Output.pm nihh20151104Brian P. Walenz A src/pipelines/canu/Output.pm nihh20151103Brian P. Walenz A src/pipelines/canu/Output.pm nihh20151102Brian P. Walenz A src/pipelines/canu/Output.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/Output.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Output.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Output.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/Output.pm bnbi20150316Brian P. Walenz D src/pipelines/canu/Output.pm src/pipelines/ca3g/Output.pm A src/pipelines/canu/Defaults.pm nihh20151119Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151118Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151113Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151108Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151104Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151103Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151022Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151021Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150921Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150911Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150903Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150826Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150809Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150729Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150701Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150615Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150610Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150603Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150529Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150529Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150529Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150528Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150526Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150509Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150508Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150429Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150421Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150420Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150410Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150327Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150327Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150327Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150325Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150317Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150316Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150310Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150306Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150303Brian P. Walenz A src/pipelines/canu/Defaults.pm bnbi20150227Brian P. Walenz D src/pipelines/canu/Defaults.pm src/pipelines/ca3g/Defaults.pm A src/pipelines/canu/OverlapStore.pm nihh20151119Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151118Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151113Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151109Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151108Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151104Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151104Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151103Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151029Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151010Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150921Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150317Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150310Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150310Brian P. Walenz A src/pipelines/canu/OverlapStore.pm bnbi20150227Brian P. Walenz D src/pipelines/canu/OverlapStore.pm src/pipelines/ca3g/OverlapStore.pm A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151119Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151113Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151109Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151108Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151105Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151104Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151103Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150921Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150528Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150526Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150420Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150310Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150306Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150306Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150303Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150303Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150303Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm bnbi20150227Brian P. Walenz D src/pipelines/canu/OverlapErrorAdjustment.pm src/pipelines/ca3g/OverlapErrorAdjustment.pm A src/pipelines/canu/OverlapBasedTrimming.pm nihh20151113Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm nihh20151109Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm nihh20151108Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm nihh20151104Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150720Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150526Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150421Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150420Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150415Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150415Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150415Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150325Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150321Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150317Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150316Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm bnbi20150316Brian P. Walenz D src/pipelines/canu/OverlapBasedTrimming.pm src/pipelines/ca3g/OverlapBasedTrimming.pm A src/pipelines/canu/OverlapMhap.pm nihh20151119Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20151109Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20151108Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20151104Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20151103Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150921Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150615Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150526Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150429Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150421Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150410Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150327Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm bnbi20150327Brian P. Walenz D src/pipelines/canu/OverlapMhap.pm src/pipelines/ca3g/OverlapMhap.pm A src/pipelines/canu/Execution.pm nihh20151119Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151119Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151113Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151113Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151113Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151108Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151104Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151103Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151103Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150911Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150828Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150723Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150615Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150529Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150526Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150421Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150420Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150327Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150317Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150310Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150303Brian P. Walenz A src/pipelines/canu/Execution.pm bnbi20150227Brian P. Walenz A src/pipelines/canu/Execution.pm jcvi20111228Brian P. Walenz A src/pipelines/canu/Execution.pm jcvi20090606Brian P. Walenz A src/pipelines/canu/Execution.pm jcvi20080925Brian P. Walenz A src/pipelines/canu/Execution.pm jcvi20060407Brian P. Walenz A src/pipelines/canu/Execution.pm none20040322Brian P. Walenz A src/pipelines/canu/Execution.pm none20040322Brian P. Walenz A src/pipelines/canu/Execution.pm none20040322Brian P. Walenz A src/pipelines/canu/Execution.pm craa20031111Brian P. Walenz A src/pipelines/canu/Execution.pm craa20030807Brian P. Walenz A src/pipelines/canu/Execution.pm craa20030729Brian P. Walenz A src/pipelines/canu/Execution.pm craa20030103Brian P. Walenz D src/pipelines/canu/Execution.pm kmer/ESTmapper/scheduler.pm D src/pipelines/canu/Execution.pm kmer/scripts/libBri.pm D src/pipelines/canu/Execution.pm kmer/scripts/scheduler.pm D src/pipelines/canu/Execution.pm src/pipelines/ca3g/Execution.pm A src/pipelines/canu/Meryl.pm nihh20151119Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20151109Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20151109Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20151108Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20151104Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20151103Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150615Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150610Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150529Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150421Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150410Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150327Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150310Brian P. Walenz A src/pipelines/canu/Meryl.pm bnbi20150227Brian P. Walenz D src/pipelines/canu/Meryl.pm src/pipelines/ca3g/Meryl.pm A src/pipelines/canu/Consensus.pm nihh20151119Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151113Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151113Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151109Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151108Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151105Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151104Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151104Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151103Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151103Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150728Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150617Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150429Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150310Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150306Brian P. Walenz A src/pipelines/canu/Consensus.pm bnbi20150306Brian P. Walenz D src/pipelines/canu/Consensus.pm src/pipelines/ca3g/Consensus.pm A src/pipelines/canu/Unitig.pm nihh20151119Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20151109Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20151108Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20151105Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20151104Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20151102Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150826Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150811Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150807Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150805Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150803Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150429Brian P. Walenz A src/pipelines/canu/Unitig.pm bnbi20150227Brian P. Walenz D src/pipelines/canu/Unitig.pm src/pipelines/ca3g/Unitig.pm A src/pipelines/canu/CorrectReads.pm nihh20151119Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20151117Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20151117Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20151117Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20151109Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20151108Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20151104Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20151103Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20151103Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20151021Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20151019Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150903Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150903Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150809Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150729Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150615Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150610Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150605Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150528Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150526Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150514Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150509Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150429Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150424Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150421Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150420Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150415Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150410Brian P. Walenz A src/pipelines/canu/CorrectReads.pm bnbi20150409Brian P. Walenz D src/pipelines/canu/CorrectReads.pm src/pipelines/ca3g/CorrectReads.pm A src/pipelines/canu/Gatekeeper.pm nihh20151109Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20151109Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20151108Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20151104Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20151027Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm bnbi20150921Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm bnbi20150825Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm bnbi20150814Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm bnbi20150616Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm bnbi20150507Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm bnbi20150429Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm bnbi20150420Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm bnbi20150410Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm bnbi20150409Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm bnbi20150227Brian P. Walenz D src/pipelines/canu/Gatekeeper.pm src/pipelines/ca3g/Gatekeeper.pm A src/pipelines/sanity/sanity-update-reference.pl nihh20151012Brian P. Walenz A src/pipelines/sanity/sanity-asm-done.pl nihh20151012Brian P. Walenz A src/pipelines/sanity/sanity-all-done.pl nihh20151012Brian P. Walenz A src/pipelines/sanity/build-all-wgs-revisions.pl nihh20151012Brian P. Walenz A src/pipelines/sanity/build-all-kmer-revisions.pl nihh20151012Brian P. Walenz A src/pipelines/sanity/sanity-purge-old.pl nihh20151012Brian P. Walenz A src/pipelines/sanity/compile-all-wgs-revisions.pl nihh20151012Brian P. Walenz A src/pipelines/sanity/sanity.pl nihh20151012Brian P. Walenz A src/pipelines/sanity/sanity-merge-qc.pl nihh20151012Brian P. Walenz A src/pipelines/sanity/sanity-get-next-date.pl nihh20151012Brian P. Walenz A src/meryl/compare-counts.C bnbi20141205Brian P. Walenz A src/meryl/compare-counts.C jcvi20140411Brian P. Walenz A src/meryl/compare-counts.C jcvi20121113Brian P. Walenz A src/meryl/compare-counts.C jcvi20120118Brian P. Walenz A src/meryl/compare-counts.C jcvi20100828Brian P. Walenz D src/meryl/compare-counts.C kmer/meryl/compare-counts.C A src/meryl/meryl-binaryOp.C bnbi20141205Brian P. Walenz A src/meryl/meryl-binaryOp.C bnbi20141205Brian P. Walenz A src/meryl/meryl-binaryOp.C jcvi20140411Brian P. Walenz A src/meryl/meryl-binaryOp.C jcvi20090804Brian P. Walenz A src/meryl/meryl-binaryOp.C jcvi20080609Brian P. Walenz A src/meryl/meryl-binaryOp.C jcvi20071012Brian P. Walenz A src/meryl/meryl-binaryOp.C jcvi20050523Brian P. Walenz A src/meryl/meryl-binaryOp.C none20041010Brian P. Walenz A src/meryl/meryl-binaryOp.C craa20040407Brian P. Walenz A src/meryl/meryl-binaryOp.C craa20040405Brian P. Walenz A src/meryl/meryl-binaryOp.C craa20030131Brian P. Walenz A src/meryl/meryl-binaryOp.C craa20030102Brian P. Walenz D src/meryl/meryl-binaryOp.C kmer/meryl/binaryOp.C A src/meryl/leaff-partition.C bnbi20141208Brian P. Walenz A src/meryl/leaff-partition.C jcvi20140411Brian P. Walenz A src/meryl/leaff-partition.C jcvi20100831Brian P. Walenz A src/meryl/leaff-partition.C jcvi20090207Brian P. Walenz D src/meryl/leaff-partition.C kmer/leaff/partition.C A src/meryl/mapMers.C bnbi20150722Brian P. Walenz A src/meryl/mapMers.C bnbi20141222Brian P. Walenz A src/meryl/mapMers.C bnbi20141205Brian P. Walenz A src/meryl/mapMers.C jcvi20140411Brian P. Walenz A src/meryl/mapMers.C jcvi20130329Brian P. Walenz A src/meryl/mapMers.C jcvi20130308Brian P. Walenz A src/meryl/mapMers.C jcvi20080911Brian P. Walenz A src/meryl/mapMers.C jcvi20080909Brian P. Walenz A src/meryl/mapMers.C jcvi20071029Brian P. Walenz A src/meryl/mapMers.C jcvi20071010Brian P. Walenz A src/meryl/mapMers.C jcvi20070913Brian P. Walenz A src/meryl/mapMers.C jcvi20070608Brian P. Walenz A src/meryl/mapMers.C jcvi20070607Brian P. Walenz A src/meryl/mapMers.C jcvi20070326Brian P. Walenz A src/meryl/mapMers.C jcvi20070320Brian P. Walenz A src/meryl/mapMers.C jcvi20061023Brian P. Walenz A src/meryl/mapMers.C jcvi20061018Brian P. Walenz D src/meryl/mapMers.C kmer/meryl/mapMers.C A src/meryl/libmeryl.H bnbi20150616Brian P. Walenz A src/meryl/libmeryl.H bnbi20141205Brian P. Walenz A src/meryl/libmeryl.H bnbi20141205Brian P. Walenz A src/meryl/libmeryl.H jcvi20140411Brian P. Walenz A src/meryl/libmeryl.H jcvi20120227Brian P. Walenz A src/meryl/libmeryl.H jcvi20080609Brian P. Walenz A src/meryl/libmeryl.H jcvi20080331Brian P. Walenz A src/meryl/libmeryl.H jcvi20080318Brian P. Walenz A src/meryl/libmeryl.H jcvi20080229Brian P. Walenz A src/meryl/libmeryl.H jcvi20080214Brian P. Walenz A src/meryl/libmeryl.H jcvi20071012Brian P. Walenz A src/meryl/libmeryl.H jcvi20051204Brian P. Walenz A src/meryl/libmeryl.H jcvi20050531Brian P. Walenz A src/meryl/libmeryl.H jcvi20050531Brian P. Walenz A src/meryl/libmeryl.H jcvi20050523Brian P. Walenz A src/meryl/libmeryl.H none20041010Brian P. Walenz A src/meryl/libmeryl.H none20040511Brian P. Walenz A src/meryl/libmeryl.H none20040430Brian P. Walenz A src/meryl/libmeryl.H none20040421Brian P. Walenz A src/meryl/libmeryl.H craa20040408Brian P. Walenz A src/meryl/libmeryl.H craa20040407Brian P. Walenz A src/meryl/libmeryl.H craa20040106Brian P. Walenz A src/meryl/libmeryl.H craa20031021Brian P. Walenz A src/meryl/libmeryl.H craa20030908Brian P. Walenz A src/meryl/libmeryl.H craa20030908Brian P. Walenz D src/meryl/libmeryl.H kmer/libmeryl/libmeryl.H A src/meryl/estimate-mer-threshold.C bnbi20141208Brian P. Walenz A src/meryl/estimate-mer-threshold.C bnbi20141205Brian P. Walenz A src/meryl/estimate-mer-threshold.C jcvi20140411Brian P. Walenz A src/meryl/estimate-mer-threshold.C jcvi20130801Brian P. Walenz A src/meryl/estimate-mer-threshold.C jcvi20130801Brian P. Walenz A src/meryl/estimate-mer-threshold.C jcvi20120306Brian P. Walenz A src/meryl/estimate-mer-threshold.C jcvi20100831Brian P. Walenz A src/meryl/estimate-mer-threshold.C jcvi20090610Brian P. Walenz A src/meryl/estimate-mer-threshold.C jcvi20081211Brian P. Walenz D src/meryl/estimate-mer-threshold.C src/AS_MER/estimate-mer-threshold.C A src/meryl/meryl-merge.C bnbi20141208Brian P. Walenz A src/meryl/meryl-merge.C bnbi20141205Brian P. Walenz A src/meryl/meryl-merge.C jcvi20140411Brian P. Walenz A src/meryl/meryl-merge.C jcvi20090807Brian P. Walenz A src/meryl/meryl-merge.C jcvi20090807Brian P. Walenz A src/meryl/meryl-merge.C jcvi20080609Brian P. Walenz A src/meryl/meryl-merge.C jcvi20080331Brian P. Walenz A src/meryl/meryl-merge.C jcvi20071012Brian P. Walenz A src/meryl/meryl-merge.C jcvi20070913Brian P. Walenz A src/meryl/meryl-merge.C jcvi20060514Brian P. Walenz A src/meryl/meryl-merge.C jcvi20050531Brian P. Walenz A src/meryl/meryl-merge.C jcvi20050523Brian P. Walenz A src/meryl/meryl-merge.C none20041010Brian P. Walenz A src/meryl/meryl-merge.C none20040729Brian P. Walenz A src/meryl/meryl-merge.C none20040726Brian P. Walenz A src/meryl/meryl-merge.C none20040409Brian P. Walenz A src/meryl/meryl-merge.C craa20040408Brian P. Walenz A src/meryl/meryl-merge.C craa20040407Brian P. Walenz A src/meryl/meryl-merge.C craa20040405Brian P. Walenz A src/meryl/meryl-merge.C craa20030131Brian P. Walenz A src/meryl/meryl-merge.C craa20030102Brian P. Walenz D src/meryl/meryl-merge.C kmer/meryl/merge.C A src/meryl/meryl.C bnbi20141208Brian P. Walenz A src/meryl/meryl.C bnbi20141205Brian P. Walenz A src/meryl/meryl.C jcvi20090807Brian P. Walenz A src/meryl/meryl.C jcvi20090804Brian P. Walenz A src/meryl/meryl.C jcvi20080909Brian P. Walenz A src/meryl/meryl.C jcvi20080609Brian P. Walenz A src/meryl/meryl.C jcvi20080318Brian P. Walenz A src/meryl/meryl.C jcvi20070608Brian P. Walenz A src/meryl/meryl.C jcvi20050523Brian P. Walenz A src/meryl/meryl.C none20041010Brian P. Walenz A src/meryl/meryl.C none20040726Brian P. Walenz A src/meryl/meryl.C none20040421Brian P. Walenz A src/meryl/meryl.C none20040414Brian P. Walenz A src/meryl/meryl.C none20040413Brian P. Walenz A src/meryl/meryl.C none20040413Brian P. Walenz A src/meryl/meryl.C craa20040407Brian P. Walenz A src/meryl/meryl.C craa20040405Brian P. Walenz A src/meryl/meryl.C none20040325Brian P. Walenz A src/meryl/meryl.C craa20031104Brian P. Walenz A src/meryl/meryl.C craa20030908Brian P. Walenz A src/meryl/meryl.C craa20030106Brian P. Walenz A src/meryl/meryl.C craa20030102Brian P. Walenz D src/meryl/meryl.C kmer/meryl/meryl.C A src/meryl/leaff-duplicates.C bnbi20141208Brian P. Walenz A src/meryl/leaff-duplicates.C jcvi20140411Brian P. Walenz A src/meryl/leaff-duplicates.C jcvi20090207Brian P. Walenz D src/meryl/leaff-duplicates.C kmer/leaff/dups.C A src/meryl/meryl-estimate.C bnbi20150529Brian P. Walenz A src/meryl/meryl-estimate.C bnbi20141208Brian P. Walenz A src/meryl/meryl-estimate.C bnbi20141205Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20140411Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20080909Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20080609Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20080401Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20071012Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20070913Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20070817Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20070326Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20060515Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20050531Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20050523Brian P. Walenz A src/meryl/meryl-estimate.C jcvi20050316Brian P. Walenz A src/meryl/meryl-estimate.C none20041010Brian P. Walenz A src/meryl/meryl-estimate.C none20040524Brian P. Walenz A src/meryl/meryl-estimate.C none20040511Brian P. Walenz A src/meryl/meryl-estimate.C none20040421Brian P. Walenz A src/meryl/meryl-estimate.C none20040413Brian P. Walenz A src/meryl/meryl-estimate.C craa20040408Brian P. Walenz A src/meryl/meryl-estimate.C craa20040407Brian P. Walenz A src/meryl/meryl-estimate.C craa20040405Brian P. Walenz A src/meryl/meryl-estimate.C none20040325Brian P. Walenz A src/meryl/meryl-estimate.C none20040323Brian P. Walenz A src/meryl/meryl-estimate.C craa20040212Clark Mobarry A src/meryl/meryl-estimate.C craa20030506Brian P. Walenz A src/meryl/meryl-estimate.C craa20030102Brian P. Walenz D src/meryl/meryl-estimate.C kmer/meryl/estimate.C A src/meryl/meryl-dump.C bnbi20141208Brian P. Walenz A src/meryl/meryl-dump.C bnbi20141205Brian P. Walenz A src/meryl/meryl-dump.C jcvi20140411Brian P. Walenz A src/meryl/meryl-dump.C jcvi20080609Brian P. Walenz A src/meryl/meryl-dump.C jcvi20080331Brian P. Walenz A src/meryl/meryl-dump.C jcvi20080318Brian P. Walenz A src/meryl/meryl-dump.C jcvi20080229Brian P. Walenz A src/meryl/meryl-dump.C jcvi20070608Brian P. Walenz A src/meryl/meryl-dump.C jcvi20070608Brian P. Walenz A src/meryl/meryl-dump.C jcvi20070606Brian P. Walenz A src/meryl/meryl-dump.C jcvi20051204Brian P. Walenz A src/meryl/meryl-dump.C jcvi20050712Brian P. Walenz A src/meryl/meryl-dump.C jcvi20050523Brian P. Walenz A src/meryl/meryl-dump.C none20041010Brian P. Walenz A src/meryl/meryl-dump.C none20040412Brian P. Walenz A src/meryl/meryl-dump.C craa20040407Brian P. Walenz A src/meryl/meryl-dump.C craa20040405Brian P. Walenz A src/meryl/meryl-dump.C craa20031021Brian P. Walenz A src/meryl/meryl-dump.C craa20030908Brian P. Walenz A src/meryl/meryl-dump.C craa20030618Brian P. Walenz A src/meryl/meryl-dump.C craa20030131Brian P. Walenz A src/meryl/meryl-dump.C craa20030102Brian P. Walenz D src/meryl/meryl-dump.C kmer/meryl/dump.C A src/meryl/leaff-statistics.C bnbi20150321Brian P. Walenz A src/meryl/leaff-statistics.C bnbi20141208Brian P. Walenz A src/meryl/leaff-statistics.C jcvi20140411Brian P. Walenz A src/meryl/leaff-statistics.C jcvi20130308Brian P. Walenz A src/meryl/leaff-statistics.C jcvi20120718Brian P. Walenz A src/meryl/leaff-statistics.C jcvi20090207Brian P. Walenz D src/meryl/leaff-statistics.C kmer/leaff/stats.C A src/meryl/libleaff/seqStore.C nihh20151029Brian P. Walenz A src/meryl/libleaff/seqStore.C bnbi20141208Brian P. Walenz A src/meryl/libleaff/seqFactory.C bnbi20150204Brian P. Walenz A src/meryl/libleaff/fastqStdin.C nihh20151029Brian P. Walenz A src/meryl/libleaff/fastqStdin.C bnbi20141208Brian P. Walenz A src/meryl/libleaff/fastaStdin.H bnbi20141208Brian P. Walenz A src/meryl/libleaff/fastaFile.C bnbi20141208Brian P. Walenz A src/meryl/libleaff/seqCache.H bnbi20141208Brian P. Walenz A src/meryl/libleaff/gkStoreFile.H nihh20151029Brian P. Walenz A src/meryl/libleaff/gkStoreFile.H bnbi20150317Brian P. Walenz A src/meryl/libleaff/gkStoreFile.H bnbi20150204Brian P. Walenz A src/meryl/libleaff/fastqFile.H bnbi20141208Brian P. Walenz A src/meryl/libleaff/seqStream.C bnbi20141208Brian P. Walenz A src/meryl/libleaff/seqFile.H nihh20151029Brian P. Walenz A src/meryl/libleaff/seqFile.H bnbi20141208Brian P. Walenz A src/meryl/libleaff/sffFile.C bnbi20141208Brian P. Walenz A src/meryl/libleaff/fastqFile.C bnbi20141208Brian P. Walenz A src/meryl/libleaff/seqCache.C bnbi20141222Brian P. Walenz A src/meryl/libleaff/seqCache.C bnbi20141208Brian P. Walenz A src/meryl/libleaff/gkStoreFile.C bnbi20150814Brian P. Walenz A src/meryl/libleaff/gkStoreFile.C bnbi20150317Brian P. Walenz A src/meryl/libleaff/gkStoreFile.C bnbi20150227Brian P. Walenz A src/meryl/libleaff/gkStoreFile.C bnbi20150204Brian P. Walenz A src/meryl/libleaff/fastaStdin.C bnbi20141208Brian P. Walenz A src/meryl/libleaff/fastaFile.H bnbi20141208Brian P. Walenz A src/meryl/libleaff/seqStore.H bnbi20141208Brian P. Walenz A src/meryl/libleaff/seqFactory.H bnbi20141208Brian P. Walenz A src/meryl/libleaff/merStream.H bnbi20141208Brian P. Walenz A src/meryl/libleaff/fastqStdin.H bnbi20141208Brian P. Walenz A src/meryl/libleaff/sffFile.H bnbi20141208Brian P. Walenz A src/meryl/libleaff/seqStream.H bnbi20141208Brian P. Walenz A src/meryl/meryl-merge-qsort.C bnbi20141205Brian P. Walenz A src/meryl/meryl-merge-qsort.C jcvi20140411Brian P. Walenz A src/meryl/meryl-merge-qsort.C jcvi20080620Brian P. Walenz D src/meryl/meryl-merge-qsort.C kmer/meryl/merge.qsort.C A src/meryl/meryl-build-threads.C bnbi20141205Brian P. Walenz A src/meryl/meryl-build-threads.C bnbi20141205Brian P. Walenz A src/meryl/meryl-build-threads.C jcvi20140411Brian P. Walenz A src/meryl/meryl-build-threads.C jcvi20061109Brian P. Walenz A src/meryl/meryl-build-threads.C jcvi20060514Brian P. Walenz A src/meryl/meryl-build-threads.C none20041010Brian P. Walenz A src/meryl/meryl-build-threads.C none20040421Brian P. Walenz A src/meryl/meryl-build-threads.C none20040413Brian P. Walenz A src/meryl/meryl-build-threads.C none20040413Brian P. Walenz D src/meryl/meryl-build-threads.C kmer/meryl/build-threads.C A src/meryl/leaff.C bnbi20150113Brian P. Walenz A src/meryl/leaff.C bnbi20141222Brian P. Walenz A src/meryl/leaff.C bnbi20141208Brian P. Walenz A src/meryl/leaff.C bnbi20141007Brian P. Walenz A src/meryl/leaff.C bnbi20140822Brian P. Walenz A src/meryl/leaff.C jcvi20140411Brian P. Walenz A src/meryl/leaff.C jcvi20120718Brian P. Walenz A src/meryl/leaff.C none20111116Liliana Florea A src/meryl/leaff.C jcvi20110330Brian P. Walenz A src/meryl/leaff.C jcvi20090707Brian P. Walenz A src/meryl/leaff.C jcvi20090613Brian P. Walenz A src/meryl/leaff.C jcvi20090416Brian P. Walenz A src/meryl/leaff.C jcvi20090413Brian P. Walenz A src/meryl/leaff.C jcvi20090207Brian P. Walenz A src/meryl/leaff.C jcvi20090128Brian P. Walenz A src/meryl/leaff.C jcvi20081023Brian P. Walenz A src/meryl/leaff.C jcvi20081023Brian P. Walenz A src/meryl/leaff.C jcvi20080922Brian P. Walenz A src/meryl/leaff.C jcvi20080912Brian P. Walenz A src/meryl/leaff.C jcvi20080909Brian P. Walenz A src/meryl/leaff.C jcvi20080626Brian P. Walenz A src/meryl/leaff.C jcvi20080508Brian P. Walenz A src/meryl/leaff.C jcvi20071220Brian P. Walenz A src/meryl/leaff.C jcvi20070407Brian P. Walenz A src/meryl/leaff.C jcvi20070326Brian P. Walenz A src/meryl/leaff.C jcvi20070320Brian P. Walenz A src/meryl/leaff.C jcvi20070320Brian P. Walenz A src/meryl/leaff.C jcvi20061220Brian P. Walenz A src/meryl/leaff.C jcvi20061211Brian P. Walenz A src/meryl/leaff.C jcvi20061022Brian P. Walenz A src/meryl/leaff.C jcvi20061012Brian P. Walenz A src/meryl/leaff.C jcvi20060808Brian P. Walenz A src/meryl/leaff.C jcvi20060728Brian P. Walenz A src/meryl/leaff.C jcvi20060718Brian P. Walenz A src/meryl/leaff.C jcvi20060706Brian P. Walenz A src/meryl/leaff.C jcvi20060407Brian P. Walenz A src/meryl/leaff.C jcvi20060228Brian P. Walenz A src/meryl/leaff.C jcvi20060207Brian P. Walenz A src/meryl/leaff.C jcvi20051211Brian P. Walenz A src/meryl/leaff.C jcvi20051012Brian P. Walenz A src/meryl/leaff.C jcvi20050916Brian P. Walenz A src/meryl/leaff.C jcvi20050727Brian P. Walenz A src/meryl/leaff.C jcvi20050522Brian P. Walenz A src/meryl/leaff.C jcvi20050506Brian P. Walenz A src/meryl/leaff.C jcvi20050323Brian P. Walenz A src/meryl/leaff.C jcvi20050306Brian P. Walenz A src/meryl/leaff.C none20041010Brian P. Walenz A src/meryl/leaff.C none20041003Brian P. Walenz A src/meryl/leaff.C none20040715Brian P. Walenz A src/meryl/leaff.C none20040624Brian P. Walenz A src/meryl/leaff.C none20040524Brian P. Walenz A src/meryl/leaff.C none20040421Brian P. Walenz A src/meryl/leaff.C none20040421Brian P. Walenz A src/meryl/leaff.C none20040312Brian P. Walenz A src/meryl/leaff.C none20040311Brian P. Walenz A src/meryl/leaff.C none20040220Brian P. Walenz A src/meryl/leaff.C craa20031014Brian P. Walenz A src/meryl/leaff.C craa20030926Brian P. Walenz A src/meryl/leaff.C craa20030805Brian P. Walenz A src/meryl/leaff.C craa20030803Brian P. Walenz A src/meryl/leaff.C craa20030721Brian P. Walenz A src/meryl/leaff.C craa20030711Brian P. Walenz A src/meryl/leaff.C craa20030710Brian P. Walenz A src/meryl/leaff.C craa20030709Brian P. Walenz A src/meryl/leaff.C craa20030514Brian P. Walenz A src/meryl/leaff.C craa20030508Brian P. Walenz A src/meryl/leaff.C craa20030502Brian P. Walenz A src/meryl/leaff.C craa20030422Brian P. Walenz A src/meryl/leaff.C craa20030415Brian P. Walenz A src/meryl/leaff.C craa20030326Brian P. Walenz A src/meryl/leaff.C craa20030310Brian P. Walenz A src/meryl/leaff.C craa20030106Brian P. Walenz A src/meryl/leaff.C craa20030102Brian P. Walenz D src/meryl/leaff.C kmer/leaff/leaff.C A src/meryl/leaff-blocks.C bnbi20141208Brian P. Walenz A src/meryl/leaff-blocks.C jcvi20140411Brian P. Walenz A src/meryl/leaff-blocks.C jcvi20090207Brian P. Walenz D src/meryl/leaff-blocks.C kmer/leaff/blocks.C A src/meryl/meryl.H bnbi20150529Brian P. Walenz A src/meryl/meryl.H bnbi20141208Brian P. Walenz A src/meryl/meryl.H bnbi20141205Brian P. Walenz A src/meryl/meryl.H bnbi20141205Brian P. Walenz A src/meryl/meryl.H jcvi20140411Brian P. Walenz A src/meryl/meryl.H jcvi20110330Brian P. Walenz A src/meryl/meryl.H jcvi20100115Brian P. Walenz A src/meryl/meryl.H jcvi20090807Brian P. Walenz A src/meryl/meryl.H jcvi20090807Brian P. Walenz A src/meryl/meryl.H jcvi20090804Brian P. Walenz A src/meryl/meryl.H jcvi20080909Brian P. Walenz A src/meryl/meryl.H jcvi20080609Brian P. Walenz A src/meryl/meryl.H jcvi20080401Brian P. Walenz A src/meryl/meryl.H jcvi20080331Brian P. Walenz A src/meryl/meryl.H jcvi20080318Brian P. Walenz A src/meryl/meryl.H jcvi20071012Brian P. Walenz A src/meryl/meryl.H jcvi20061023Brian P. Walenz A src/meryl/meryl.H jcvi20060515Brian P. Walenz A src/meryl/meryl.H jcvi20060514Brian P. Walenz A src/meryl/meryl.H jcvi20050531Brian P. Walenz A src/meryl/meryl.H jcvi20050523Brian P. Walenz A src/meryl/meryl.H none20041010Brian P. Walenz A src/meryl/meryl.H none20040726Brian P. Walenz A src/meryl/meryl.H none20040421Brian P. Walenz A src/meryl/meryl.H none20040414Brian P. Walenz A src/meryl/meryl.H none20040413Brian P. Walenz A src/meryl/meryl.H craa20040407Brian P. Walenz A src/meryl/meryl.H craa20040405Brian P. Walenz A src/meryl/meryl.H none20040325Brian P. Walenz A src/meryl/meryl.H craa20030908Brian P. Walenz A src/meryl/meryl.H craa20030102Brian P. Walenz D src/meryl/meryl.H kmer/meryl/meryl.H A src/meryl/leaff-simulate.C nihh20151012Brian P. Walenz A src/meryl/leaff-simulate.C bnbi20141208Brian P. Walenz A src/meryl/leaff-simulate.C jcvi20090613Brian P. Walenz A src/meryl/leaff-simulate.C jcvi20080922Brian P. Walenz A src/meryl/leaff-simulate.C none20041010Brian P. Walenz A src/meryl/leaff-simulate.C none20040715Brian P. Walenz A src/meryl/leaff-simulate.C none20040629Brian P. Walenz A src/meryl/leaff-simulate.C none20040624Brian P. Walenz D src/meryl/leaff-simulate.C kmer/leaff/simseq.C A src/meryl/gkrpt.pl nihh20151012Brian P. Walenz A src/meryl/gkrpt.pl nihh20151012Brian P. Walenz A src/meryl/gkrpt.pl bnbi20150410Brian P. Walenz A src/meryl/gkrpt.pl jcvi20130801Brian P. Walenz A src/meryl/gkrpt.pl jcvi20080627Brian P. Walenz A src/meryl/gkrpt.pl jcvi20080103Brian P. Walenz D src/meryl/gkrpt.pl src/AS_MER/gkrpt.pl A src/meryl/meryl-args.C bnbi20150529Brian P. Walenz A src/meryl/meryl-args.C bnbi20141222Brian P. Walenz A src/meryl/meryl-args.C bnbi20141205Brian P. Walenz A src/meryl/meryl-args.C bnbi20141205Brian P. Walenz A src/meryl/meryl-args.C jcvi20140411Brian P. Walenz A src/meryl/meryl-args.C jcvi20110330Brian P. Walenz A src/meryl/meryl-args.C jcvi20100331Brian P. Walenz A src/meryl/meryl-args.C jcvi20100115Brian P. Walenz A src/meryl/meryl-args.C jcvi20090807Brian P. Walenz A src/meryl/meryl-args.C jcvi20090804Brian P. Walenz A src/meryl/meryl-args.C jcvi20090724Brian P. Walenz A src/meryl/meryl-args.C jcvi20080925Brian P. Walenz A src/meryl/meryl-args.C jcvi20080909Brian P. Walenz A src/meryl/meryl-args.C jcvi20080609Brian P. Walenz A src/meryl/meryl-args.C jcvi20080401Brian P. Walenz A src/meryl/meryl-args.C jcvi20080331Brian P. Walenz A src/meryl/meryl-args.C jcvi20080318Brian P. Walenz A src/meryl/meryl-args.C jcvi20071012Brian P. Walenz A src/meryl/meryl-args.C jcvi20070608Brian P. Walenz A src/meryl/meryl-args.C jcvi20070407Brian P. Walenz A src/meryl/meryl-args.C jcvi20061218Brian P. Walenz A src/meryl/meryl-args.C jcvi20061109Brian P. Walenz A src/meryl/meryl-args.C jcvi20061105Brian P. Walenz A src/meryl/meryl-args.C jcvi20060527Brian P. Walenz A src/meryl/meryl-args.C jcvi20060515Brian P. Walenz A src/meryl/meryl-args.C jcvi20060514Brian P. Walenz A src/meryl/meryl-args.C jcvi20050531Brian P. Walenz A src/meryl/meryl-args.C jcvi20050523Brian P. Walenz A src/meryl/meryl-args.C jcvi20050320Brian P. Walenz A src/meryl/meryl-args.C none20041010Brian P. Walenz A src/meryl/meryl-args.C none20040726Brian P. Walenz A src/meryl/meryl-args.C craa20040701Brian P. Walenz A src/meryl/meryl-args.C none20040421Brian P. Walenz A src/meryl/meryl-args.C none20040414Brian P. Walenz A src/meryl/meryl-args.C none20040413Brian P. Walenz A src/meryl/meryl-args.C none20040412Brian P. Walenz A src/meryl/meryl-args.C craa20040407Brian P. Walenz A src/meryl/meryl-args.C craa20040405Brian P. Walenz A src/meryl/meryl-args.C none20040331Brian P. Walenz D src/meryl/meryl-args.C kmer/meryl/args.C A src/meryl/libmeryl.C nihh20151029Brian P. Walenz A src/meryl/libmeryl.C bnbi20150701Brian P. Walenz A src/meryl/libmeryl.C bnbi20150616Brian P. Walenz A src/meryl/libmeryl.C bnbi20141205Brian P. Walenz A src/meryl/libmeryl.C bnbi20141205Brian P. Walenz A src/meryl/libmeryl.C jcvi20140411Brian P. Walenz A src/meryl/libmeryl.C jcvi20120227Brian P. Walenz A src/meryl/libmeryl.C jcvi20080609Brian P. Walenz A src/meryl/libmeryl.C jcvi20080331Brian P. Walenz A src/meryl/libmeryl.C jcvi20080318Brian P. Walenz A src/meryl/libmeryl.C jcvi20080229Brian P. Walenz A src/meryl/libmeryl.C jcvi20071012Brian P. Walenz A src/meryl/libmeryl.C jcvi20070827Brian P. Walenz A src/meryl/libmeryl.C jcvi20060524Brian P. Walenz A src/meryl/libmeryl.C jcvi20050531Brian P. Walenz A src/meryl/libmeryl.C jcvi20050531Brian P. Walenz A src/meryl/libmeryl.C jcvi20050523Brian P. Walenz A src/meryl/libmeryl.C none20040430Brian P. Walenz A src/meryl/libmeryl.C none20040430Brian P. Walenz A src/meryl/libmeryl.C craa20040408Brian P. Walenz A src/meryl/libmeryl.C craa20040407Brian P. Walenz A src/meryl/libmeryl.C craa20040405Brian P. Walenz A src/meryl/libmeryl.C craa20040106Brian P. Walenz A src/meryl/libmeryl.C craa20031104Brian P. Walenz A src/meryl/libmeryl.C craa20030908Brian P. Walenz D src/meryl/libmeryl.C kmer/libmeryl/libmeryl.C A src/meryl/maskMers.C bnbi20141205Brian P. Walenz A src/meryl/maskMers.C jcvi20140411Brian P. Walenz A src/meryl/maskMers.C jcvi20080909Brian P. Walenz A src/meryl/maskMers.C jcvi20080430Brian P. Walenz A src/meryl/maskMers.C jcvi20080429Brian P. Walenz A src/meryl/maskMers.C jcvi20080429Brian P. Walenz A src/meryl/maskMers.C jcvi20080429Brian P. Walenz A src/meryl/maskMers.C jcvi20080429Brian P. Walenz A src/meryl/maskMers.C jcvi20080428Brian P. Walenz A src/meryl/maskMers.C jcvi20080421Brian P. Walenz A src/meryl/maskMers.C jcvi20080402Brian P. Walenz A src/meryl/maskMers.C jcvi20080402Brian P. Walenz A src/meryl/maskMers.C jcvi20080401Brian P. Walenz A src/meryl/maskMers.C jcvi20080331Brian P. Walenz A src/meryl/maskMers.C jcvi20080331Brian P. Walenz D src/meryl/maskMers.C kmer/meryl/maskMers.C A src/meryl/leaff-gc.C bnbi20141208Brian P. Walenz A src/meryl/leaff-gc.C jcvi20140411Brian P. Walenz A src/meryl/leaff-gc.C jcvi20090207Brian P. Walenz D src/meryl/leaff-gc.C kmer/leaff/gc.C A src/meryl/meryl-build.C bnbi20150701Brian P. Walenz A src/meryl/meryl-build.C bnbi20150529Brian P. Walenz A src/meryl/meryl-build.C bnbi20141208Brian P. Walenz A src/meryl/meryl-build.C bnbi20141205Brian P. Walenz A src/meryl/meryl-build.C bnbi20141205Brian P. Walenz A src/meryl/meryl-build.C jcvi20140411Brian P. Walenz A src/meryl/meryl-build.C jcvi20121015Brian P. Walenz A src/meryl/meryl-build.C jcvi20111214Brian P. Walenz A src/meryl/meryl-build.C jcvi20110330Brian P. Walenz A src/meryl/meryl-build.C jcvi20090724Brian P. Walenz A src/meryl/meryl-build.C jcvi20090721Brian P. Walenz A src/meryl/meryl-build.C jcvi20081012Brian P. Walenz A src/meryl/meryl-build.C jcvi20081012Brian P. Walenz A src/meryl/meryl-build.C jcvi20080917Brian P. Walenz A src/meryl/meryl-build.C jcvi20080909Brian P. Walenz A src/meryl/meryl-build.C jcvi20080609Brian P. Walenz A src/meryl/meryl-build.C jcvi20080401Brian P. Walenz A src/meryl/meryl-build.C jcvi20080331Brian P. Walenz A src/meryl/meryl-build.C jcvi20080318Brian P. Walenz A src/meryl/meryl-build.C jcvi20071208Brian P. Walenz A src/meryl/meryl-build.C jcvi20071023Brian P. Walenz A src/meryl/meryl-build.C jcvi20071012Brian P. Walenz A src/meryl/meryl-build.C jcvi20070913Brian P. Walenz A src/meryl/meryl-build.C jcvi20070827Brian P. Walenz A src/meryl/meryl-build.C jcvi20070823Brian P. Walenz A src/meryl/meryl-build.C jcvi20070823Brian P. Walenz A src/meryl/meryl-build.C jcvi20070606Brian P. Walenz A src/meryl/meryl-build.C jcvi20070330Brian P. Walenz A src/meryl/meryl-build.C jcvi20070326Brian P. Walenz A src/meryl/meryl-build.C jcvi20061109Brian P. Walenz A src/meryl/meryl-build.C jcvi20060527Brian P. Walenz A src/meryl/meryl-build.C jcvi20060515Brian P. Walenz A src/meryl/meryl-build.C jcvi20060514Brian P. Walenz A src/meryl/meryl-build.C jcvi20050911Brian P. Walenz A src/meryl/meryl-build.C jcvi20050531Brian P. Walenz A src/meryl/meryl-build.C jcvi20050531Brian P. Walenz A src/meryl/meryl-build.C jcvi20050523Brian P. Walenz A src/meryl/meryl-build.C jcvi20050320Brian P. Walenz A src/meryl/meryl-build.C jcvi20050312Brian P. Walenz A src/meryl/meryl-build.C none20041010Brian P. Walenz A src/meryl/meryl-build.C none20040812Brian P. Walenz A src/meryl/meryl-build.C none20040623Brian P. Walenz A src/meryl/meryl-build.C none20040421Brian P. Walenz A src/meryl/meryl-build.C none20040413Brian P. Walenz A src/meryl/meryl-build.C none20040413Brian P. Walenz A src/meryl/meryl-build.C none20040413Brian P. Walenz A src/meryl/meryl-build.C none20040409Brian P. Walenz A src/meryl/meryl-build.C craa20040408Brian P. Walenz A src/meryl/meryl-build.C craa20040407Brian P. Walenz A src/meryl/meryl-build.C craa20040405Brian P. Walenz A src/meryl/meryl-build.C none20040325Brian P. Walenz A src/meryl/meryl-build.C none20040323Brian P. Walenz A src/meryl/meryl-build.C craa20040212Clark Mobarry A src/meryl/meryl-build.C craa20030506Brian P. Walenz A src/meryl/meryl-build.C craa20030131Brian P. Walenz A src/meryl/meryl-build.C craa20030102Brian P. Walenz D src/meryl/meryl-build.C kmer/meryl/build.C A src/meryl/simple.C bnbi20150424Brian P. Walenz A src/meryl/simple.C bnbi20150421Brian P. Walenz A src/meryl/simple.C bnbi20141205Brian P. Walenz A src/meryl/simple.C jcvi20140411Brian P. Walenz A src/meryl/simple.C jcvi20110330Brian P. Walenz A src/meryl/simple.C jcvi20100831Brian P. Walenz D src/meryl/simple.C kmer/meryl/simple.C A src/meryl/mapMers-depth.C bnbi20141222Brian P. Walenz A src/meryl/mapMers-depth.C bnbi20141205Brian P. Walenz A src/meryl/mapMers-depth.C bnbi20141014Brian P. Walenz A src/meryl/mapMers-depth.C bnbi20141007Brian P. Walenz A src/meryl/mapMers-depth.C jcvi20140411Brian P. Walenz A src/meryl/mapMers-depth.C jcvi20080911Brian P. Walenz A src/meryl/mapMers-depth.C jcvi20080909Brian P. Walenz A src/meryl/mapMers-depth.C jcvi20071029Brian P. Walenz A src/meryl/mapMers-depth.C jcvi20071010Brian P. Walenz A src/meryl/mapMers-depth.C jcvi20070913Brian P. Walenz A src/meryl/mapMers-depth.C jcvi20070619Brian P. Walenz A src/meryl/mapMers-depth.C jcvi20070608Brian P. Walenz D src/meryl/mapMers-depth.C kmer/meryl/mapMers-depth.C A src/meryl/meryl-merge-listmerge.C bnbi20141205Brian P. Walenz A src/meryl/meryl-merge-listmerge.C jcvi20140411Brian P. Walenz A src/meryl/meryl-merge-listmerge.C jcvi20080620Brian P. Walenz D src/meryl/meryl-merge-listmerge.C kmer/meryl/merge.listmerge.C A src/meryl/meryl-unaryOp.C bnbi20141205Brian P. Walenz A src/meryl/meryl-unaryOp.C jcvi20080609Brian P. Walenz A src/meryl/meryl-unaryOp.C jcvi20080331Brian P. Walenz A src/meryl/meryl-unaryOp.C jcvi20071012Brian P. Walenz A src/meryl/meryl-unaryOp.C none20041010Brian P. Walenz A src/meryl/meryl-unaryOp.C craa20040407Brian P. Walenz A src/meryl/meryl-unaryOp.C craa20040405Brian P. Walenz A src/meryl/meryl-unaryOp.C craa20030131Brian P. Walenz A src/meryl/meryl-unaryOp.C craa20030102Brian P. Walenz D src/meryl/meryl-unaryOp.C kmer/meryl/unaryOp.C A src/mhap/mhapConvert.C bnbi20150625Brian P. Walenz A src/mhap/mhapConvert.C bnbi20150616Brian P. Walenz A src/mhap/mhapConvert.C bnbi20150409Brian P. Walenz A src/mhap/mhapConvert.C bnbi20150327Brian P. Walenz A src/stores/tgTig.H nihh20151113Brian P. Walenz A src/stores/tgTig.H nihh20151030Brian P. Walenz A src/stores/tgTig.H bnbi20150814Brian P. Walenz A src/stores/tgTig.H bnbi20150811Brian P. Walenz A src/stores/tgTig.H bnbi20150807Brian P. Walenz A src/stores/tgTig.H bnbi20150708Brian P. Walenz A src/stores/tgTig.H bnbi20150701Brian P. Walenz A src/stores/tgTig.H bnbi20150623Brian P. Walenz A src/stores/tgTig.H bnbi20150603Brian P. Walenz A src/stores/tgTig.H bnbi20150507Brian P. Walenz A src/stores/tgTig.H bnbi20150409Brian P. Walenz A src/stores/tgTig.H bnbi20150303Brian P. Walenz A src/stores/tgTig.H bnbi20150127Brian P. Walenz A src/stores/tgTig.H bnbi20150127Brian P. Walenz A src/stores/tgTig.H bnbi20150127Brian P. Walenz A src/stores/tgTig.H bnbi20150121Brian P. Walenz A src/stores/tgTig.H bnbi20150113Brian P. Walenz A src/stores/tgTig.H bnbi20150113Brian P. Walenz A src/stores/tgTig.H bnbi20150110Brian P. Walenz A src/stores/tgTig.H bnbi20141222Brian P. Walenz A src/stores/gkStore.H nihh20151029Brian P. Walenz A src/stores/gkStore.H nihh20151009Brian P. Walenz A src/stores/gkStore.H bnbi20150814Brian P. Walenz A src/stores/gkStore.H bnbi20150810Brian P. Walenz A src/stores/gkStore.H bnbi20150810Brian P. Walenz A src/stores/gkStore.H bnbi20150701Brian P. Walenz A src/stores/gkStore.H bnbi20150605Brian P. Walenz A src/stores/gkStore.H bnbi20150603Brian P. Walenz A src/stores/gkStore.H bnbi20150529Brian P. Walenz A src/stores/gkStore.H bnbi20150529Brian P. Walenz A src/stores/gkStore.H bnbi20150514Brian P. Walenz A src/stores/gkStore.H bnbi20150508Brian P. Walenz A src/stores/gkStore.H bnbi20150318Brian P. Walenz A src/stores/gkStore.H bnbi20150317Brian P. Walenz A src/stores/gkStore.H bnbi20150317Brian P. Walenz A src/stores/gkStore.H bnbi20150306Brian P. Walenz A src/stores/gkStore.H bnbi20150227Brian P. Walenz A src/stores/gkStore.H bnbi20150227Brian P. Walenz A src/stores/gkStore.H bnbi20150224Brian P. Walenz A src/stores/gkStore.H bnbi20150113Brian P. Walenz A src/stores/gkStore.H bnbi20141223Brian P. Walenz A src/stores/gkStore.H bnbi20141215Brian P. Walenz A src/stores/gkStore.H bnbi20141202Brian P. Walenz A src/stores/gkStore.H bnbi20141127Brian P. Walenz A src/stores/gkStore.H bnbi20141126Brian P. Walenz A src/stores/ovStore.C nihh20151012Brian P. Walenz A src/stores/ovStore.C nihh20151012Brian P. Walenz A src/stores/ovStore.C bnbi20150814Brian P. Walenz A src/stores/ovStore.C bnbi20150701Brian P. Walenz A src/stores/ovStore.C bnbi20150625Brian P. Walenz A src/stores/ovStore.C bnbi20150616Brian P. Walenz A src/stores/ovStore.C bnbi20150616Brian P. Walenz A src/stores/ovStore.C bnbi20150605Brian P. Walenz A src/stores/ovStore.C bnbi20150520Brian P. Walenz A src/stores/ovStore.C bnbi20150514Brian P. Walenz A src/stores/ovStore.C bnbi20150413Brian P. Walenz A src/stores/ovStore.C bnbi20150409Brian P. Walenz A src/stores/ovStore.C bnbi20150321Brian P. Walenz A src/stores/ovStore.C bnbi20150317Brian P. Walenz A src/stores/ovStore.C bnbi20150312Brian P. Walenz A src/stores/ovStore.C bnbi20150306Brian P. Walenz A src/stores/ovStore.C bnbi20150227Brian P. Walenz A src/stores/ovStore.C bnbi20150113Brian P. Walenz A src/stores/ovStore.C bnbi20141223Brian P. Walenz A src/stores/ovStore.C bnbi20141215Brian P. Walenz A src/stores/ovStore.C bnbi20141209Brian P. Walenz A src/stores/ovStore.C jcvi20130801Brian P. Walenz A src/stores/ovStore.C jcvi20130801Brian P. Walenz A src/stores/ovStore.C jcvi20130801Brian P. Walenz A src/stores/ovStore.C jcvi20130702Brian P. Walenz A src/stores/ovStore.C jcvi20130702Brian P. Walenz A src/stores/ovStore.C jcvi20130111Brian P. Walenz A src/stores/ovStore.C jcvi20120424Brian P. Walenz A src/stores/ovStore.C jcvi20120321Brian P. Walenz A src/stores/ovStore.C jcvi20120214Gregory Sims A src/stores/ovStore.C jcvi20120208Gregory Sims A src/stores/ovStore.C jcvi20120203Gregory Sims A src/stores/ovStore.C jcvi20120201Gregory Sims A src/stores/ovStore.C jcvi20111205Brian P. Walenz A src/stores/ovStore.C jcvi20110603Brian P. Walenz A src/stores/ovStore.C bnbi20110603Sergey Koren A src/stores/ovStore.C bnbi20110602Sergey Koren A src/stores/ovStore.C jcvi20110211Brian P. Walenz A src/stores/ovStore.C jcvi20101215Brian P. Walenz A src/stores/ovStore.C jcvi20091201Brian P. Walenz A src/stores/ovStore.C jcvi20091026Brian P. Walenz A src/stores/ovStore.C jcvi20091026Brian P. Walenz A src/stores/ovStore.C jcvi20090712Brian P. Walenz A src/stores/ovStore.C jcvi20090610Brian P. Walenz A src/stores/ovStore.C jcvi20081008Brian P. Walenz A src/stores/ovStore.C jcvi20080627Brian P. Walenz A src/stores/ovStore.C jcvi20080103Brian P. Walenz A src/stores/ovStore.C jcvi20071127Brian P. Walenz A src/stores/ovStore.C jcvi20071120Brian P. Walenz A src/stores/ovStore.C jcvi20071119Brian P. Walenz A src/stores/ovStore.C jcvi20070603Brian P. Walenz A src/stores/ovStore.C jcvi20070510Brian P. Walenz A src/stores/ovStore.C jcvi20070508Sergey Koren A src/stores/ovStore.C jcvi20070416Brian P. Walenz A src/stores/ovStore.C jcvi20070402Brian P. Walenz A src/stores/ovStore.C jcvi20070316Brian P. Walenz A src/stores/ovStore.C jcvi20070313Brian P. Walenz A src/stores/ovStore.C jcvi20070313Brian P. Walenz A src/stores/ovStore.C jcvi20070312Brian P. Walenz A src/stores/ovStore.C jcvi20070309Brian P. Walenz A src/stores/ovStore.C jcvi20070309Brian P. Walenz A src/stores/ovStore.C jcvi20070308Brian P. Walenz A src/stores/ovStore.C jcvi20070308Brian P. Walenz D src/stores/ovStore.C src/AS_OVS/AS_OVS_overlapStore.C D src/stores/ovStore.C src/AS_OVS/AS_OVS_overlapStore.c A src/stores/gatekeeperDumpMetaData.C bnbi20150921Brian P. Walenz A src/stores/gatekeeperDumpMetaData.C bnbi20150701Brian P. Walenz A src/stores/gatekeeperDumpMetaData.C bnbi20150603Brian P. Walenz A src/stores/gatekeeperDumpMetaData.C bnbi20150529Brian P. Walenz A src/stores/gatekeeperDumpMetaData.C bnbi20150507Brian P. Walenz A src/stores/gatekeeperDumpMetaData.C bnbi20150321Brian P. Walenz A src/stores/gatekeeperDumpMetaData.C bnbi20150318Brian P. Walenz A src/stores/ovStoreBucketizer.C nihh20151108Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20150921Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20150814Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20150625Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20150616Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20150514Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20150410Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20150321Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20150317Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20150310Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20150224Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20141215Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20141121Brian P. Walenz A src/stores/ovStoreBucketizer.C bnbi20140822Brian P. Walenz A src/stores/ovStoreBucketizer.C jcvi20130801Brian P. Walenz A src/stores/ovStoreBucketizer.C jcvi20130801Brian P. Walenz A src/stores/ovStoreBucketizer.C jcvi20120602Brian P. Walenz A src/stores/ovStoreBucketizer.C jcvi20120510Brian P. Walenz A src/stores/ovStoreBucketizer.C jcvi20120508Brian P. Walenz A src/stores/ovStoreBucketizer.C jcvi20120417Brian P. Walenz A src/stores/ovStoreBucketizer.C jcvi20120417Brian P. Walenz A src/stores/ovStoreBucketizer.C jcvi20120403Brian P. Walenz A src/stores/ovStoreBucketizer.C jcvi20120402Brian P. Walenz A src/stores/ovStoreBucketizer.C jcvi20120402Brian P. Walenz D src/stores/ovStoreBucketizer.C src/AS_OVS/overlapStoreBucketizer.C A src/stores/ovStoreDump.C nihh20151021Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150625Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150616Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150616Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150605Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150526Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150520Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150514Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150429Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150409Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150321Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150317Brian P. Walenz A src/stores/ovStoreDump.C bnbi20150317Brian P. Walenz A src/stores/ovStoreDump.C bnbi20141215Brian P. Walenz A src/stores/ovStoreDump.C bnbi20141210Brian P. Walenz A src/stores/ovStoreDump.C bnbi20141021Brian P. Walenz A src/stores/ovStoreDump.C bnbi20140822Brian P. Walenz A src/stores/ovStoreDump.C jcvi20130801Brian P. Walenz A src/stores/ovStoreDump.C jcvi20130801Brian P. Walenz A src/stores/ovStoreDump.C jcvi20130801Brian P. Walenz A src/stores/ovStoreDump.C jcvi20120508Brian P. Walenz A src/stores/ovStoreDump.C jcvi20120402Brian P. Walenz A src/stores/ovStoreDump.C jcvi20120314Gregory Sims A src/stores/ovStoreDump.C jcvi20120203Gregory Sims A src/stores/ovStoreDump.C jcvi20120201Gregory Sims A src/stores/ovStoreDump.C jcvi20111229Brian P. Walenz A src/stores/ovStoreDump.C jcvi20110824Brian P. Walenz A src/stores/ovStoreDump.C jcvi20110102Brian P. Walenz A src/stores/ovStoreDump.C jcvi20101001Brian P. Walenz A src/stores/ovStoreDump.C jcvi20100928Brian P. Walenz A src/stores/ovStoreDump.C jcvi20100416Brian P. Walenz A src/stores/ovStoreDump.C jcvi20100129Brian P. Walenz A src/stores/ovStoreDump.C jcvi20091104Brian P. Walenz A src/stores/ovStoreDump.C jcvi20091026Brian P. Walenz A src/stores/ovStoreDump.C jcvi20090814Sergey Koren A src/stores/ovStoreDump.C jcvi20090706Brian P. Walenz A src/stores/ovStoreDump.C jcvi20090514Brian P. Walenz A src/stores/ovStoreDump.C jcvi20090225Brian P. Walenz A src/stores/ovStoreDump.C jcvi20081216Sergey Koren A src/stores/ovStoreDump.C jcvi20081113Brian P. Walenz A src/stores/ovStoreDump.C jcvi20081013Brian P. Walenz A src/stores/ovStoreDump.C jcvi20081008Brian P. Walenz A src/stores/ovStoreDump.C jcvi20080627Brian P. Walenz A src/stores/ovStoreDump.C jcvi20071119Brian P. Walenz A src/stores/ovStoreDump.C jcvi20071108Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070802Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070723Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070723Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070412Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070403Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070313Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070312Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070309Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070309Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070308Brian P. Walenz A src/stores/ovStoreDump.C jcvi20070308Brian P. Walenz D src/stores/ovStoreDump.C src/AS_OVS/overlapStore.C D src/stores/ovStoreDump.C src/AS_OVS/overlapStore.c A src/stores/ovStoreIndexer.C bnbi20150921Brian P. Walenz A src/stores/ovStoreIndexer.C bnbi20150312Brian P. Walenz A src/stores/ovStoreIndexer.C bnbi20150310Brian P. Walenz A src/stores/ovStoreIndexer.C bnbi20141215Brian P. Walenz A src/stores/ovStoreIndexer.C jcvi20130801Brian P. Walenz A src/stores/ovStoreIndexer.C jcvi20130801Brian P. Walenz A src/stores/ovStoreIndexer.C jcvi20120602Brian P. Walenz A src/stores/ovStoreIndexer.C jcvi20120516Brian P. Walenz A src/stores/ovStoreIndexer.C jcvi20120508Brian P. Walenz A src/stores/ovStoreIndexer.C jcvi20120424Brian P. Walenz A src/stores/ovStoreIndexer.C jcvi20120417Brian P. Walenz A src/stores/ovStoreIndexer.C jcvi20120403Brian P. Walenz A src/stores/ovStoreIndexer.C jcvi20120402Brian P. Walenz D src/stores/ovStoreIndexer.C src/AS_OVS/overlapStoreIndexer.C A src/stores/ovStoreSorter.C bnbi20150921Brian P. Walenz A src/stores/ovStoreSorter.C bnbi20150625Brian P. Walenz A src/stores/ovStoreSorter.C bnbi20150616Brian P. Walenz A src/stores/ovStoreSorter.C bnbi20150312Brian P. Walenz A src/stores/ovStoreSorter.C bnbi20150310Brian P. Walenz A src/stores/ovStoreSorter.C bnbi20141215Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20130909Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20130801Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20130801Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20120819Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20120602Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20120508Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20120417Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20120417Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20120417Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20120417Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20120403Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20120402Brian P. Walenz A src/stores/ovStoreSorter.C jcvi20120402Brian P. Walenz D src/stores/ovStoreSorter.C src/AS_OVS/overlapStoreSorter.C A src/stores/gatekeeperDumpFASTQ.C bnbi20150603Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20150415Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20150410Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20150327Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20150321Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20150318Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20150317Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20150317Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20150227Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20141215Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20141215Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20141202Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20141127Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C bnbi20141121Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C jcvi20130801Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C jcvi20130801Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C jcvi20120206Brian P. Walenz D src/stores/gatekeeperDumpFASTQ.C src/AS_GKP/gkpStoreDumpFASTQ.C A src/stores/gatekeeperPartition.C bnbi20150317Brian P. Walenz A src/stores/gatekeeperPartition.C bnbi20150306Brian P. Walenz A src/stores/gatekeeperPartition.C bnbi20150306Brian P. Walenz A src/stores/gatekeeperPartition.C bnbi20150305Brian P. Walenz A src/stores/gatekeeperPartition.C bnbi20141223Brian P. Walenz A src/stores/tgStore.H nihh20151030Brian P. Walenz A src/stores/tgStore.H nihh20151029Brian P. Walenz A src/stores/tgStore.H bnbi20150811Brian P. Walenz A src/stores/tgStore.H bnbi20150701Brian P. Walenz A src/stores/tgStore.H bnbi20150113Brian P. Walenz A src/stores/tgStore.H bnbi20150113Brian P. Walenz A src/stores/tgStore.H bnbi20141222Brian P. Walenz A src/stores/tgStore.H jcvi20140331Brian P. Walenz A src/stores/tgStore.H jcvi20130918Brian P. Walenz A src/stores/tgStore.H jcvi20130801Brian P. Walenz A src/stores/tgStore.H jcvi20130801Brian P. Walenz A src/stores/tgStore.H jcvi20130801Brian P. Walenz A src/stores/tgStore.H jcvi20121114Brian P. Walenz A src/stores/tgStore.H jcvi20120412Brian P. Walenz A src/stores/tgStore.H jcvi20120328Brian P. Walenz A src/stores/tgStore.H jcvi20110102Brian P. Walenz A src/stores/tgStore.H jcvi20100923Brian P. Walenz A src/stores/tgStore.H jcvi20091212Brian P. Walenz A src/stores/tgStore.H jcvi20091210Brian P. Walenz A src/stores/tgStore.H jcvi20091209Brian P. Walenz A src/stores/tgStore.H jcvi20091119Brian P. Walenz A src/stores/tgStore.H jcvi20091007Brian P. Walenz A src/stores/tgStore.H jcvi20091005Brian P. Walenz A src/stores/tgStore.H jcvi20091005Brian P. Walenz D src/stores/tgStore.H src/AS_CNS/MultiAlignStore.H D src/stores/tgStore.H src/AS_CNS/MultiAlignStore.h A src/stores/tgStoreLoad.C bnbi20150814Brian P. Walenz A src/stores/tgStoreLoad.C bnbi20150811Brian P. Walenz A src/stores/tgStoreLoad.C bnbi20150807Brian P. Walenz A src/stores/tgTigSizeAnalysis.C nihh20151030Brian P. Walenz A src/stores/tgTigSizeAnalysis.C bnbi20141222Brian P. Walenz A src/stores/tgTigSizeAnalysis.C jcvi20131024Brian P. Walenz A src/stores/tgTigSizeAnalysis.C jcvi20130801Brian P. Walenz A src/stores/tgTigSizeAnalysis.C jcvi20120626Brian P. Walenz A src/stores/tgTigSizeAnalysis.C jcvi20120326Brian P. Walenz D src/stores/tgTigSizeAnalysis.C src/AS_CNS/MultiAlignSizeAnalysis.C A src/stores/gatekeeperCreate.C nihh20151108Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150810Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150729Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150715Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150701Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150603Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150603Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150514Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150429Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150325Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150317Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150303Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150227Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150204Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150204Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20150130Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20141223Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20141215Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20141202Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20141127Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20141121Brian P. Walenz A src/stores/gatekeeperCreate.C bnbi20141009Brian P. Walenz A src/stores/gatekeeperCreate.C jcvi20130801Brian P. Walenz A src/stores/gatekeeperCreate.C jcvi20130801Brian P. Walenz A src/stores/gatekeeperCreate.C jcvi20120206Brian P. Walenz D src/stores/gatekeeperCreate.C src/AS_GKP/gkpStoreCreate.C A src/stores/ovStoreStats.C nihh20151104Brian P. Walenz A src/stores/ovStoreStats.C nihh20151029Brian P. Walenz A src/stores/ovStoreStats.C nihh20151027Brian P. Walenz A src/stores/ovStoreStats.C nihh20151027Brian P. Walenz A src/stores/ovStore.H nihh20151027Brian P. Walenz A src/stores/ovStore.H nihh20151027Brian P. Walenz A src/stores/ovStore.H nihh20151026Brian P. Walenz A src/stores/ovStore.H nihh20151012Brian P. Walenz A src/stores/ovStore.H bnbi20150701Brian P. Walenz A src/stores/ovStore.H bnbi20150625Brian P. Walenz A src/stores/ovStore.H bnbi20150616Brian P. Walenz A src/stores/ovStore.H bnbi20150616Brian P. Walenz A src/stores/ovStore.H bnbi20150605Brian P. Walenz A src/stores/ovStore.H bnbi20150529Brian P. Walenz A src/stores/ovStore.H bnbi20150528Brian P. Walenz A src/stores/ovStore.H bnbi20150514Brian P. Walenz A src/stores/ovStore.H bnbi20150413Brian P. Walenz A src/stores/ovStore.H bnbi20150409Brian P. Walenz A src/stores/ovStore.H bnbi20150321Brian P. Walenz A src/stores/ovStore.H bnbi20150317Brian P. Walenz A src/stores/ovStore.H bnbi20150317Brian P. Walenz A src/stores/ovStore.H bnbi20150306Brian P. Walenz A src/stores/ovStore.H bnbi20150211Brian P. Walenz A src/stores/ovStore.H bnbi20150113Brian P. Walenz A src/stores/ovStore.H bnbi20141230Brian P. Walenz A src/stores/ovStore.H bnbi20141223Brian P. Walenz A src/stores/ovStore.H bnbi20141215Brian P. Walenz A src/stores/ovStore.H bnbi20141209Brian P. Walenz A src/stores/tgStoreDump.C nihh20151108Brian P. Walenz A src/stores/tgStoreDump.C nihh20151104Brian P. Walenz A src/stores/tgStoreDump.C nihh20151103Brian P. Walenz A src/stores/tgStoreDump.C nihh20151102Brian P. Walenz A src/stores/tgStoreDump.C nihh20151030Brian P. Walenz A src/stores/tgStoreDump.C nihh20151012Brian P. Walenz A src/stores/tgStoreDump.C bnbi20150814Brian P. Walenz A src/stores/tgStoreDump.C bnbi20150811Brian P. Walenz A src/stores/tgStoreDump.C bnbi20150807Brian P. Walenz A src/stores/tgStoreDump.C bnbi20150701Brian P. Walenz A src/stores/tgStoreDump.C bnbi20150603Brian P. Walenz A src/stores/tgStoreDump.C bnbi20150526Brian P. Walenz A src/stores/tgStoreDump.C bnbi20150509Brian P. Walenz A src/stores/tgStoreDump.C bnbi20150409Brian P. Walenz A src/stores/tgStoreDump.C bnbi20150113Brian P. Walenz A src/stores/tgStoreDump.C bnbi20150113Brian P. Walenz A src/stores/tgStoreDump.C bnbi20141222Brian P. Walenz A src/stores/tgStoreDump.C bnbi20141014Brian P. Walenz A src/stores/tgStoreDump.C bnbi20141009Brian P. Walenz A src/stores/tgStoreDump.C bnbi20140413Sergey Koren A src/stores/tgStoreDump.C jcvi20140331Brian P. Walenz A src/stores/tgStoreDump.C jcvi20140225Brian P. Walenz A src/stores/tgStoreDump.C jcvi20140225Brian P. Walenz A src/stores/tgStoreDump.C jcvi20140220Brian P. Walenz A src/stores/tgStoreDump.C jcvi20140218Brian P. Walenz A src/stores/tgStoreDump.C jcvi20140123Brian P. Walenz A src/stores/tgStoreDump.C jcvi20130918Brian P. Walenz A src/stores/tgStoreDump.C jcvi20130917Brian P. Walenz A src/stores/tgStoreDump.C jcvi20130917Brian P. Walenz A src/stores/tgStoreDump.C jcvi20130916Brian P. Walenz A src/stores/tgStoreDump.C jcvi20130801Brian P. Walenz A src/stores/tgStoreDump.C jcvi20130801Brian P. Walenz A src/stores/tgStoreDump.C jcvi20130503Brian P. Walenz A src/stores/tgStoreDump.C jcvi20130131Brian P. Walenz A src/stores/tgStoreDump.C jcvi20120910Brian P. Walenz A src/stores/tgStoreDump.C jcvi20120810Brian P. Walenz A src/stores/tgStoreDump.C jcvi20120806Brian P. Walenz A src/stores/tgStoreDump.C jcvi20120417Brian P. Walenz A src/stores/tgStoreDump.C jcvi20120412Brian P. Walenz A src/stores/tgStoreDump.C jcvi20120328Brian P. Walenz A src/stores/tgStoreDump.C jcvi20120326Brian P. Walenz A src/stores/tgStoreDump.C jcvi20111218Brian P. Walenz A src/stores/tgStoreDump.C jcvi20111218Brian P. Walenz A src/stores/tgStoreDump.C jcvi20111215Brian P. Walenz A src/stores/tgStoreDump.C jcvi20111208Brian P. Walenz A src/stores/tgStoreDump.C jcvi20111207Brian P. Walenz A src/stores/tgStoreDump.C jcvi20110815Brian P. Walenz A src/stores/tgStoreDump.C jcvi20110102Brian P. Walenz A src/stores/tgStoreDump.C jcvi20100920Brian P. Walenz A src/stores/tgStoreDump.C tigr20100517Michael Schatz A src/stores/tgStoreDump.C jcvi20100423Brian P. Walenz A src/stores/tgStoreDump.C jcvi20100402Brian P. Walenz A src/stores/tgStoreDump.C jcvi20100329Brian P. Walenz A src/stores/tgStoreDump.C tigr20100227Michael Schatz A src/stores/tgStoreDump.C jcvi20091212Brian P. Walenz A src/stores/tgStoreDump.C jcvi20091209Brian P. Walenz A src/stores/tgStoreDump.C jcvi20091104Brian P. Walenz A src/stores/tgStoreDump.C jcvi20091007Brian P. Walenz A src/stores/tgStoreDump.C jcvi20091005Brian P. Walenz D src/stores/tgStoreDump.C src/AS_CNS/tigStore.C A src/stores/tgTig.C nihh20151113Brian P. Walenz A src/stores/tgTig.C nihh20151030Brian P. Walenz A src/stores/tgTig.C nihh20151030Brian P. Walenz A src/stores/tgTig.C bnbi20150811Brian P. Walenz A src/stores/tgTig.C bnbi20150807Brian P. Walenz A src/stores/tgTig.C bnbi20150701Brian P. Walenz A src/stores/tgTig.C bnbi20150603Brian P. Walenz A src/stores/tgTig.C bnbi20150507Brian P. Walenz A src/stores/tgTig.C bnbi20150507Brian P. Walenz A src/stores/tgTig.C bnbi20150409Brian P. Walenz A src/stores/tgTig.C bnbi20150306Brian P. Walenz A src/stores/tgTig.C bnbi20150127Brian P. Walenz A src/stores/tgTig.C bnbi20150127Brian P. Walenz A src/stores/tgTig.C bnbi20150126Brian P. Walenz A src/stores/tgTig.C bnbi20150121Brian P. Walenz A src/stores/tgTig.C bnbi20150113Brian P. Walenz A src/stores/tgTig.C bnbi20150110Brian P. Walenz A src/stores/tgTig.C bnbi20141222Brian P. Walenz A src/stores/tgStoreFilter.C nihh20151104Brian P. Walenz A src/stores/tgStoreFilter.C nihh20151030Brian P. Walenz A src/stores/tgStoreFilter.C bnbi20150814Brian P. Walenz A src/stores/tgStoreFilter.C bnbi20141014Brian P. Walenz A src/stores/tgStoreFilter.C bnbi20141009Brian P. Walenz A src/stores/tgStoreFilter.C jcvi20140415Brian P. Walenz A src/stores/tgStoreFilter.C jcvi20140415Brian P. Walenz A src/stores/tgStoreFilter.C bnbi20140414Sergey Koren A src/stores/tgStoreFilter.C jcvi20140331Brian P. Walenz D src/stores/tgStoreFilter.C src/AS_BAT/markRepeatUnique.C A src/stores/tgTigDisplay.C bnbi20150701Brian P. Walenz A src/stores/tgTigDisplay.C bnbi20150409Brian P. Walenz A src/stores/tgTigDisplay.C bnbi20150126Brian P. Walenz A src/stores/gkStore.C nihh20151029Brian P. Walenz A src/stores/gkStore.C nihh20151021Brian P. Walenz A src/stores/gkStore.C nihh20151009Brian P. Walenz A src/stores/gkStore.C bnbi20150810Brian P. Walenz A src/stores/gkStore.C bnbi20150810Brian P. Walenz A src/stores/gkStore.C bnbi20150729Brian P. Walenz A src/stores/gkStore.C bnbi20150701Brian P. Walenz A src/stores/gkStore.C bnbi20150529Brian P. Walenz A src/stores/gkStore.C bnbi20150526Brian P. Walenz A src/stores/gkStore.C bnbi20150507Brian P. Walenz A src/stores/gkStore.C bnbi20150429Brian P. Walenz A src/stores/gkStore.C bnbi20150327Brian P. Walenz A src/stores/gkStore.C bnbi20150317Brian P. Walenz A src/stores/gkStore.C bnbi20150317Brian P. Walenz A src/stores/gkStore.C bnbi20150306Brian P. Walenz A src/stores/gkStore.C bnbi20150306Brian P. Walenz A src/stores/gkStore.C bnbi20150303Brian P. Walenz A src/stores/gkStore.C bnbi20150227Brian P. Walenz A src/stores/gkStore.C bnbi20150227Brian P. Walenz A src/stores/gkStore.C bnbi20150224Brian P. Walenz A src/stores/gkStore.C bnbi20141223Brian P. Walenz A src/stores/gkStore.C bnbi20141219Brian P. Walenz A src/stores/gkStore.C bnbi20141215Brian P. Walenz A src/stores/gkStore.C bnbi20141215Brian P. Walenz A src/stores/gkStore.C bnbi20141202Brian P. Walenz A src/stores/gkStore.C bnbi20141127Brian P. Walenz A src/stores/gkStore.C bnbi20141126Brian P. Walenz A src/stores/ovOverlap.C bnbi20150701Brian P. Walenz A src/stores/ovOverlap.C bnbi20150616Brian P. Walenz A src/stores/ovOverlap.C bnbi20150616Brian P. Walenz A src/stores/ovOverlap.C bnbi20150605Brian P. Walenz A src/stores/ovOverlap.C bnbi20150520Brian P. Walenz A src/stores/ovOverlap.C bnbi20150514Brian P. Walenz A src/stores/ovOverlap.C bnbi20150409Brian P. Walenz A src/stores/ovOverlap.C bnbi20150321Brian P. Walenz A src/stores/ovOverlap.C bnbi20150127Brian P. Walenz A src/stores/ovOverlap.C bnbi20141215Brian P. Walenz A src/stores/tgStore.C nihh20151030Brian P. Walenz A src/stores/tgStore.C nihh20151029Brian P. Walenz A src/stores/tgStore.C bnbi20150811Brian P. Walenz A src/stores/tgStore.C bnbi20150701Brian P. Walenz A src/stores/tgStore.C bnbi20150409Brian P. Walenz A src/stores/tgStore.C bnbi20150127Brian P. Walenz A src/stores/tgStore.C bnbi20150126Brian P. Walenz A src/stores/tgStore.C bnbi20150113Brian P. Walenz A src/stores/tgStore.C bnbi20150113Brian P. Walenz A src/stores/tgStore.C bnbi20141222Brian P. Walenz A src/stores/tgStore.C jcvi20140331Brian P. Walenz A src/stores/tgStore.C jcvi20131018Brian P. Walenz A src/stores/tgStore.C jcvi20130918Brian P. Walenz A src/stores/tgStore.C jcvi20130905Brian P. Walenz A src/stores/tgStore.C jcvi20130801Brian P. Walenz A src/stores/tgStore.C jcvi20130801Brian P. Walenz A src/stores/tgStore.C jcvi20121116Brian P. Walenz A src/stores/tgStore.C jcvi20121114Brian P. Walenz A src/stores/tgStore.C jcvi20121112Brian P. Walenz A src/stores/tgStore.C jcvi20120914Brian P. Walenz A src/stores/tgStore.C jcvi20120913Brian P. Walenz A src/stores/tgStore.C jcvi20120829Brian P. Walenz A src/stores/tgStore.C jcvi20120810Brian P. Walenz A src/stores/tgStore.C jcvi20120412Brian P. Walenz A src/stores/tgStore.C jcvi20120328Brian P. Walenz A src/stores/tgStore.C jcvi20110506Brian P. Walenz A src/stores/tgStore.C jcvi20100923Brian P. Walenz A src/stores/tgStore.C jcvi20100824Brian P. Walenz A src/stores/tgStore.C jcvi20100414Brian P. Walenz A src/stores/tgStore.C jcvi20100205Sergey Koren A src/stores/tgStore.C jcvi20100121Brian P. Walenz A src/stores/tgStore.C jcvi20100116Brian P. Walenz A src/stores/tgStore.C jcvi20091212Brian P. Walenz A src/stores/tgStore.C jcvi20091210Brian P. Walenz A src/stores/tgStore.C jcvi20091210Brian P. Walenz A src/stores/tgStore.C jcvi20091209Brian P. Walenz A src/stores/tgStore.C jcvi20091209Brian P. Walenz A src/stores/tgStore.C jcvi20091203Brian P. Walenz A src/stores/tgStore.C jcvi20091202Brian P. Walenz A src/stores/tgStore.C jcvi20091119Brian P. Walenz A src/stores/tgStore.C jcvi20091012Brian P. Walenz A src/stores/tgStore.C jcvi20091007Brian P. Walenz A src/stores/tgStore.C jcvi20091007Brian P. Walenz A src/stores/tgStore.C jcvi20091007Brian P. Walenz A src/stores/tgStore.C jcvi20091005Brian P. Walenz D src/stores/tgStore.C src/AS_CNS/MultiAlignStore.C A src/stores/tgTigSizeAnalysis.H nihh20151030Brian P. Walenz A src/stores/tgTigSizeAnalysis.H bnbi20141222Brian P. Walenz A src/stores/tgTigSizeAnalysis.H jcvi20130801Brian P. Walenz A src/stores/tgTigSizeAnalysis.H jcvi20130801Brian P. Walenz A src/stores/tgTigSizeAnalysis.H jcvi20120626Brian P. Walenz A src/stores/tgTigSizeAnalysis.H jcvi20120326Brian P. Walenz D src/stores/tgTigSizeAnalysis.H src/AS_CNS/MultiAlignSizeAnalysis.H A src/stores/tgTigMultiAlignDisplay.C bnbi20150603Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C bnbi20150409Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C bnbi20150317Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C bnbi20150131Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C bnbi20150128Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C bnbi20150127Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C bnbi20150127Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C bnbi20150126Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C bnbi20150113Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C bnbi20141121Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20130801Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20130801Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20130801Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20111208Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20111207Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20111206Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20111110Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20110102Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20100806Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20091005Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20090907Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20090610Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20081008Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20080627Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20071108Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C jcvi20070905Brian P. Walenz D src/stores/tgTigMultiAlignDisplay.C src/AS_CNS/MultiAlignPrint.C D src/stores/tgTigMultiAlignDisplay.C src/AS_CNS/MultiAlignPrint.c A src/stores/ovStoreBuild.C nihh20151118Brian P. Walenz A src/stores/ovStoreBuild.C nihh20151108Brian P. Walenz A src/stores/ovStoreBuild.C nihh20151103Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150921Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150814Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150625Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150616Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150616Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150514Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150321Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150317Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150306Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150303Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20150224Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20141215Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20141121Brian P. Walenz A src/stores/ovStoreBuild.C bnbi20140731Brian P. Walenz A src/stores/ovStoreBuild.C jcvi20130801Brian P. Walenz A src/stores/ovStoreBuild.C jcvi20130801Brian P. Walenz A src/stores/ovStoreBuild.C jcvi20120819Brian P. Walenz A src/stores/ovStoreBuild.C jcvi20120602Brian P. Walenz A src/stores/ovStoreBuild.C jcvi20120515Brian P. Walenz A src/stores/ovStoreBuild.C jcvi20120508Brian P. Walenz A src/stores/ovStoreBuild.C jcvi20120403Brian P. Walenz A src/stores/ovStoreBuild.C jcvi20120402Brian P. Walenz D src/stores/ovStoreBuild.C src/AS_OVS/overlapStoreBuild.C A src/stores/tgStoreCoverageStat.C nihh20151104Brian P. Walenz A src/stores/tgStoreCoverageStat.C nihh20151103Brian P. Walenz A src/stores/tgStoreCoverageStat.C nihh20151030Brian P. Walenz A src/stores/tgStoreCoverageStat.C nihh20151012Brian P. Walenz A src/stores/tgStoreCoverageStat.C bnbi20150814Brian P. Walenz A src/stores/tgStoreCoverageStat.C jcvi20130801Brian P. Walenz A src/stores/tgStoreCoverageStat.C jcvi20130801Brian P. Walenz A src/stores/tgStoreCoverageStat.C jcvi20120829Brian P. Walenz A src/stores/tgStoreCoverageStat.C jcvi20120718Jason Miller A src/stores/tgStoreCoverageStat.C jcvi20120327Brian P. Walenz A src/stores/tgStoreCoverageStat.C jcvi20120313Brian P. Walenz A src/stores/tgStoreCoverageStat.C jcvi20120115Brian P. Walenz A src/stores/tgStoreCoverageStat.C jcvi20111230Brian P. Walenz A src/stores/tgStoreCoverageStat.C jcvi20111229Brian P. Walenz A src/stores/tgStoreCoverageStat.C jcvi20111220Brian P. Walenz D src/stores/tgStoreCoverageStat.C src/AS_BAT/computeCoverageStat.C A src/stores/ovStoreFile.C bnbi20150616Brian P. Walenz A src/stores/ovStoreFile.C bnbi20150529Brian P. Walenz A src/stores/ovStoreFile.C bnbi20150409Brian P. Walenz A src/stores/ovStoreFile.C bnbi20150306Brian P. Walenz A src/stores/ovStoreFile.C bnbi20150211Brian P. Walenz A src/stores/ovStoreFile.C bnbi20141215Brian P. Walenz A src/stores/ovStoreFile.C bnbi20141215Brian P. Walenz A src/stores/ovStoreFile.C bnbi20141209Brian P. Walenz A src/stores/ovStoreFile.C jcvi20130922Brian P. Walenz A src/stores/ovStoreFile.C jcvi20130801Brian P. Walenz A src/stores/ovStoreFile.C jcvi20130801Brian P. Walenz A src/stores/ovStoreFile.C jcvi20130801Brian P. Walenz A src/stores/ovStoreFile.C jcvi20120602Brian P. Walenz A src/stores/ovStoreFile.C jcvi20120508Brian P. Walenz A src/stores/ovStoreFile.C jcvi20120201Gregory Sims A src/stores/ovStoreFile.C jcvi20110401Brian P. Walenz A src/stores/ovStoreFile.C bnbi20110331Sergey Koren A src/stores/ovStoreFile.C jcvi20091026Brian P. Walenz A src/stores/ovStoreFile.C jcvi20090410Brian P. Walenz A src/stores/ovStoreFile.C jcvi20090222Brian P. Walenz A src/stores/ovStoreFile.C jcvi20081013Brian P. Walenz A src/stores/ovStoreFile.C jcvi20081008Brian P. Walenz A src/stores/ovStoreFile.C jcvi20080627Brian P. Walenz A src/stores/ovStoreFile.C jcvi20070723Brian P. Walenz A src/stores/ovStoreFile.C jcvi20070603Brian P. Walenz A src/stores/ovStoreFile.C jcvi20070320Brian P. Walenz A src/stores/ovStoreFile.C jcvi20070313Brian P. Walenz A src/stores/ovStoreFile.C jcvi20070312Brian P. Walenz A src/stores/ovStoreFile.C jcvi20070309Brian P. Walenz A src/stores/ovStoreFile.C jcvi20070309Brian P. Walenz A src/stores/ovStoreFile.C jcvi20070308Brian P. Walenz A src/stores/ovStoreFile.C jcvi20070308Brian P. Walenz A src/stores/ovStoreFile.C jcvi20070228Brian P. Walenz D src/stores/ovStoreFile.C src/AS_OVS/AS_OVS_overlapFile.C D src/stores/ovStoreFile.C src/AS_OVS/AS_OVS_overlapFile.c A src/bogart-analysis/show-false-best-edges-from-mapping.pl nihh20151012Brian P. Walenz A src/bogart-analysis/locate-read-in-unitig-based-on-olaps.pl nihh20151012Brian P. Walenz A src/bogart-analysis/examine-mapping.pl nihh20151012Brian P. Walenz A src/bogart-analysis/count-bubbles.pl nihh20151012Brian P. Walenz A src/bogart-analysis/examine-mapping-ideal.pl nihh20151012Brian P. Walenz A src/bogart-analysis/bogart-build-fasta.pl nihh20151012Brian P. Walenz A src/bogart-analysis/analyze-nucmer-gaps.pl nihh20151012Brian P. Walenz A src/bogart-analysis/analyze-mapped-unitigs-for-joins.pl nihh20151012Brian P. Walenz A src/AS_global.H nihh20151029Brian P. Walenz A src/AS_global.H bnbi20150331Brian P. Walenz A src/AS_global.H bnbi20150303Brian P. Walenz A src/AS_global.H bnbi20150227Brian P. Walenz A src/AS_global.H bnbi20141222Brian P. Walenz A src/AS_global.H bnbi20141209Brian P. Walenz A src/AS_global.H bnbi20141208Brian P. Walenz A src/AS_global.H bnbi20141205Brian P. Walenz A src/AS_global.H bnbi20141126Brian P. Walenz A src/AS_global.H bnbi20141121Brian P. Walenz A src/AS_global.H bnbi20141117Brian P. Walenz A src/AS_global.H bnbi20140811Brian P. Walenz A src/AS_global.H jcvi20140305Brian P. Walenz A src/AS_global.H jcvi20131106Brian P. Walenz A src/AS_global.H jcvi20130801Brian P. Walenz A src/AS_global.H jcvi20130801Brian P. Walenz A src/AS_global.H jcvi20130801Brian P. Walenz A src/AS_global.H jcvi20130501Brian P. Walenz A src/AS_global.H jcvi20130222Brian P. Walenz A src/AS_global.H jcvi20120721Brian P. Walenz A src/AS_global.H jcvi20120508Brian P. Walenz A src/AS_global.H jcvi20120203Brian P. Walenz A src/AS_global.H jcvi20111229Brian P. Walenz A src/AS_global.H jcvi20111204Brian P. Walenz A src/AS_global.H bnbi20111011Sergey Koren A src/AS_global.H bnbi20110502Sergey Koren A src/AS_global.H jcvi20110125Brian P. Walenz A src/AS_global.H jcvi20110106Brian P. Walenz A src/AS_global.H jcvi20101025Brian P. Walenz A src/AS_global.H jcvi20091027Brian P. Walenz A src/AS_global.H jcvi20091026Brian P. Walenz A src/AS_global.H jcvi20090610Brian P. Walenz A src/AS_global.H jcvi20090202Brian P. Walenz A src/AS_global.H jcvi20081111Brian P. Walenz A src/AS_global.H jcvi20081009Brian P. Walenz A src/AS_global.H jcvi20081008Brian P. Walenz A src/AS_global.H jcvi20080627Brian P. Walenz A src/AS_global.H jcvi20080514Brian P. Walenz A src/AS_global.H jcvi20080514Brian P. Walenz A src/AS_global.H jcvi20071108Brian P. Walenz A src/AS_global.H jcvi20071024Brian P. Walenz A src/AS_global.H jcvi20071024Brian P. Walenz A src/AS_global.H jcvi20070810Brian P. Walenz A src/AS_global.H jcvi20070804Brian P. Walenz A src/AS_global.H jcvi20070803Brian P. Walenz A src/AS_global.H jcvi20070725Brian P. Walenz A src/AS_global.H jcvi20070723Brian P. Walenz A src/AS_global.H jcvi20070723Brian P. Walenz A src/AS_global.H jcvi20070603Brian P. Walenz A src/AS_global.H jcvi20070514Brian P. Walenz A src/AS_global.H jcvi20070416Brian P. Walenz A src/AS_global.H jcvi20070319Brian P. Walenz A src/AS_global.H jcvi20070318Brian P. Walenz A src/AS_global.H jcvi20070308Brian P. Walenz A src/AS_global.H jcvi20070225Brian P. Walenz A src/AS_global.H jcvi20070224Brian P. Walenz A src/AS_global.H jcvi20070223Brian P. Walenz A src/AS_global.H jcvi20061218Brian P. Walenz A src/AS_global.H jcvi20061218Brian P. Walenz A src/AS_global.H jcvi20061114Eli Venter A src/AS_global.H jcvi20061114Eli Venter A src/AS_global.H jcvi20061008Brian P. Walenz A src/AS_global.H jcvi20061008Brian P. Walenz A src/merTrim/merTrimApply.C bnbi20141205Brian P. Walenz A src/merTrim/merTrimApply.C bnbi20141009Brian P. Walenz A src/merTrim/merTrimApply.C jcvi20130801Brian P. Walenz A src/merTrim/merTrimApply.C jcvi20130801Brian P. Walenz A src/merTrim/merTrimApply.C jcvi20111229Brian P. Walenz A src/merTrim/merTrimApply.C jcvi20110822Brian P. Walenz A src/merTrim/merTrimApply.C jcvi20110603Brian P. Walenz A src/merTrim/merTrimApply.C jcvi20110106Brian P. Walenz A src/merTrim/merTrimApply.C jcvi20110104Brian P. Walenz D src/merTrim/merTrimApply.C src/AS_MER/merTrimApply.C A src/merTrim/merTrim-compare-logs.pl nihh20151012Brian P. Walenz A src/merTrim/merTrim-compare-logs.pl bnbi20141205Brian P. Walenz A src/merTrim/merTrim-compare-logs.pl bnbi20141115Brian P. Walenz D src/merTrim/merTrim-compare-logs.pl src/AS_MER/merTrim-compare-logs.pl A src/merTrim/merTrim.C bnbi20141205Brian P. Walenz A src/merTrim/merTrim.C jcvi20140411Brian P. Walenz A src/merTrim/merTrim.C jcvi20131009Brian P. Walenz A src/merTrim/merTrim.C jcvi20130922Brian P. Walenz A src/merTrim/merTrim.C jcvi20130801Brian P. Walenz A src/merTrim/merTrim.C jcvi20130801Brian P. Walenz A src/merTrim/merTrim.C jcvi20130620Brian P. Walenz A src/merTrim/merTrim.C jcvi20130508Brian P. Walenz A src/merTrim/merTrim.C jcvi20130402Brian P. Walenz A src/merTrim/merTrim.C jcvi20130329Brian P. Walenz A src/merTrim/merTrim.C jcvi20130308Brian P. Walenz A src/merTrim/merTrim.C jcvi20121119Brian P. Walenz A src/merTrim/merTrim.C jcvi20121114Brian P. Walenz A src/merTrim/merTrim.C jcvi20121101Brian P. Walenz A src/merTrim/merTrim.C jcvi20121023Brian P. Walenz A src/merTrim/merTrim.C jcvi20120726Brian P. Walenz A src/merTrim/merTrim.C jcvi20120529Brian P. Walenz A src/merTrim/merTrim.C jcvi20120521Brian P. Walenz A src/merTrim/merTrim.C jcvi20120510Brian P. Walenz A src/merTrim/merTrim.C jcvi20120323Brian P. Walenz A src/merTrim/merTrim.C jcvi20120227Brian P. Walenz A src/merTrim/merTrim.C jcvi20120227Brian P. Walenz A src/merTrim/merTrim.C jcvi20120226Brian P. Walenz A src/merTrim/merTrim.C jcvi20120212Brian P. Walenz A src/merTrim/merTrim.C jcvi20120202Brian P. Walenz A src/merTrim/merTrim.C jcvi20120131Brian P. Walenz A src/merTrim/merTrim.C jcvi20120131Brian P. Walenz A src/merTrim/merTrim.C jcvi20120131Brian P. Walenz A src/merTrim/merTrim.C jcvi20120131Brian P. Walenz A src/merTrim/merTrim.C jcvi20120124Brian P. Walenz A src/merTrim/merTrim.C jcvi20120123Brian P. Walenz A src/merTrim/merTrim.C jcvi20120123Brian P. Walenz A src/merTrim/merTrim.C jcvi20120118Brian P. Walenz A src/merTrim/merTrim.C jcvi20120117Brian P. Walenz A src/merTrim/merTrim.C jcvi20111229Brian P. Walenz A src/merTrim/merTrim.C jcvi20110822Brian P. Walenz A src/merTrim/merTrim.C jcvi20110728Brian P. Walenz A src/merTrim/merTrim.C jcvi20110721Brian P. Walenz A src/merTrim/merTrim.C jcvi20110708Brian P. Walenz A src/merTrim/merTrim.C jcvi20110623Brian P. Walenz A src/merTrim/merTrim.C jcvi20110603Brian P. Walenz A src/merTrim/merTrim.C jcvi20110104Brian P. Walenz A src/merTrim/merTrim.C jcvi20101102Brian P. Walenz A src/merTrim/merTrim.C jcvi20100712Sergey Koren A src/merTrim/merTrim.C jcvi20100325Brian P. Walenz A src/merTrim/merTrim.C jcvi20100322Brian P. Walenz A src/merTrim/merTrim.C jcvi20100321Brian P. Walenz A src/merTrim/merTrim.C jcvi20100316Brian P. Walenz A src/merTrim/merTrim.C jcvi20100301Brian P. Walenz A src/merTrim/merTrim.C jcvi20100222Brian P. Walenz D src/merTrim/merTrim.C src/AS_MER/merTrim.C A src/merTrim/merTrimAdapter.C bnbi20141205Brian P. Walenz A src/merTrim/merTrimAdapter.C jcvi20130801Brian P. Walenz A src/merTrim/merTrimAdapter.C jcvi20130801Brian P. Walenz A src/merTrim/merTrimAdapter.C jcvi20120521Brian P. Walenz A src/merTrim/merTrimAdapter.C jcvi20120510Brian P. Walenz D src/merTrim/merTrimAdapter.C src/AS_MER/merTrimAdapter.C A src/merTrim/merTrimResult.H bnbi20141205Brian P. Walenz A src/merTrim/merTrimResult.H jcvi20130801Brian P. Walenz A src/merTrim/merTrimResult.H jcvi20130801Brian P. Walenz A src/merTrim/merTrimResult.H jcvi20110822Brian P. Walenz D src/merTrim/merTrimResult.H src/AS_MER/merTrimResult.H A src/bogus/bogusness.C bnbi20141223Brian P. Walenz A src/bogus/bogusness.C bnbi20141219Brian P. Walenz A src/bogus/bogusness.C jcvi20130801Brian P. Walenz A src/bogus/bogusness.C jcvi20110815Brian P. Walenz A src/bogus/bogusness.C jcvi20110506Brian P. Walenz A src/bogus/bogusness.C jcvi20110404Brian P. Walenz A src/bogus/bogusness.C jcvi20110210Brian P. Walenz A src/bogus/bogusness.C jcvi20101221Brian P. Walenz A src/bogus/bogusness.C jcvi20101217Brian P. Walenz A src/bogus/bogusness.C jcvi20101213Brian P. Walenz A src/bogus/bogusness.C jcvi20101206Brian P. Walenz A src/bogus/bogusness.C jcvi20101202Brian P. Walenz A src/bogus/bogusness.C jcvi20101123Brian P. Walenz D src/bogus/bogusness.C src/AS_BAT/bogusness.C A src/bogus/bogusUtil.C bnbi20141223Brian P. Walenz A src/bogus/bogusUtil.C bnbi20141219Brian P. Walenz A src/bogus/bogusUtil.C bnbi20141219Brian P. Walenz A src/bogus/bogusUtil.C jcvi20140123Brian P. Walenz A src/bogus/bogusUtil.C jcvi20130801Brian P. Walenz A src/bogus/bogusUtil.C jcvi20130417Brian P. Walenz A src/bogus/bogusUtil.C jcvi20130329Brian P. Walenz A src/bogus/bogusUtil.C jcvi20130315Brian P. Walenz A src/bogus/bogusUtil.C jcvi20110815Brian P. Walenz A src/bogus/bogusUtil.C jcvi20110627Jason Miller A src/bogus/bogusUtil.C jcvi20110404Brian P. Walenz A src/bogus/bogusUtil.C jcvi20110210Brian P. Walenz A src/bogus/bogusUtil.C jcvi20101221Brian P. Walenz A src/bogus/bogusUtil.C jcvi20101217Brian P. Walenz A src/bogus/bogusUtil.C jcvi20101213Brian P. Walenz A src/bogus/bogusUtil.C jcvi20101213Brian P. Walenz A src/bogus/bogusUtil.C jcvi20101206Brian P. Walenz A src/bogus/bogusUtil.C jcvi20101206Brian P. Walenz A src/bogus/bogusUtil.C jcvi20101202Brian P. Walenz A src/bogus/bogusUtil.C jcvi20101123Brian P. Walenz D src/bogus/bogusUtil.C src/AS_BAT/AS_BAT_bogusUtil.C D src/bogus/bogusUtil.C src/bogart/AS_BAT_bogusUtil.C A src/bogus/bogusness-run.pl nihh20151012Brian P. Walenz A src/bogus/bogusness-run.pl bnbi20141219Brian P. Walenz A src/bogus/bogusness-run.pl jcvi20130801Brian P. Walenz A src/bogus/bogusness-run.pl jcvi20110606Brian P. Walenz A src/bogus/bogusness-run.pl jcvi20110601Brian P. Walenz A src/bogus/bogusness-run.pl jcvi20110506Brian P. Walenz A src/bogus/bogusness-run.pl jcvi20110404Brian P. Walenz A src/bogus/bogusness-run.pl jcvi20110317Brian P. Walenz A src/bogus/bogusness-run.pl jcvi20101213Brian P. Walenz A src/bogus/bogusness-run.pl jcvi20101202Brian P. Walenz D src/bogus/bogusness-run.pl src/AS_BAT/bogusness-run.pl A src/bogus/bogus.C bnbi20141223Brian P. Walenz A src/bogus/bogus.C bnbi20141219Brian P. Walenz A src/bogus/bogus.C bnbi20141014Brian P. Walenz A src/bogus/bogus.C bnbi20141009Brian P. Walenz A src/bogus/bogus.C jcvi20130801Brian P. Walenz A src/bogus/bogus.C jcvi20130417Brian P. Walenz A src/bogus/bogus.C jcvi20111229Brian P. Walenz A src/bogus/bogus.C jcvi20110815Brian P. Walenz A src/bogus/bogus.C jcvi20110726Brian P. Walenz A src/bogus/bogus.C jcvi20110404Brian P. Walenz A src/bogus/bogus.C jcvi20110210Brian P. Walenz A src/bogus/bogus.C jcvi20101221Brian P. Walenz A src/bogus/bogus.C jcvi20101217Brian P. Walenz A src/bogus/bogus.C jcvi20101213Brian P. Walenz A src/bogus/bogus.C jcvi20101206Brian P. Walenz A src/bogus/bogus.C jcvi20101123Brian P. Walenz D src/bogus/bogus.C src/AS_BAT/bogus.C A src/bogus/bogusUtil.H bnbi20141223Brian P. Walenz A src/bogus/bogusUtil.H bnbi20141219Brian P. Walenz A src/bogus/bogusUtil.H bnbi20141219Brian P. Walenz A src/bogus/bogusUtil.H bnbi20141009Brian P. Walenz A src/bogus/bogusUtil.H jcvi20130801Brian P. Walenz A src/bogus/bogusUtil.H jcvi20130801Brian P. Walenz A src/bogus/bogusUtil.H jcvi20111229Brian P. Walenz A src/bogus/bogusUtil.H jcvi20110815Brian P. Walenz A src/bogus/bogusUtil.H jcvi20110404Brian P. Walenz A src/bogus/bogusUtil.H jcvi20110210Brian P. Walenz A src/bogus/bogusUtil.H jcvi20101221Brian P. Walenz A src/bogus/bogusUtil.H jcvi20101217Brian P. Walenz A src/bogus/bogusUtil.H jcvi20101213Brian P. Walenz A src/bogus/bogusUtil.H jcvi20101206Brian P. Walenz A src/bogus/bogusUtil.H jcvi20101123Brian P. Walenz D src/bogus/bogusUtil.H src/AS_BAT/AS_BAT_bogusUtil.H D src/bogus/bogusUtil.H src/bogart/AS_BAT_bogusUtil.H A src/AS_RUN/replaceIIDwithName-overlapDump.pl nihh20151012Brian P. Walenz A src/AS_RUN/replaceIIDwithName-overlapDump.pl jcvi20130922Brian P. Walenz A src/AS_RUN/replaceUIDwithName-fastq.pl nihh20151012Brian P. Walenz A src/AS_RUN/replaceUIDwithName-fastq.pl bnbi20141001Brian P. Walenz A src/AS_RUN/replaceUIDwithName-fastq.pl bnbi20141001Brian P. Walenz A src/AS_RUN/replaceUIDwithName-fastq.pl jcvi20130823Brian P. Walenz A src/AS_RUN/replaceUIDwithName-fastq.pl jcvi20130801Brian P. Walenz A src/AS_RUN/replaceUIDwithName-fastq.pl jcvi20130308Brian P. Walenz A src/AS_RUN/replaceUIDwithName-fastq.pl jcvi20130131Brian P. Walenz A src/AS_RUN/replaceUIDwithName-fastq.pl jcvi20121213Brian P. Walenz D src/AS_RUN/replaceUIDwithName-fastq.pl src/AS_RUN/replaceUIDwithName.pl A src/AS_RUN/fragmentDepth.C bnbi20141121Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20130801Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20130801Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20130801Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20120812Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20120508Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20100322Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20090814Sergey Koren A src/AS_RUN/fragmentDepth.C jcvi20090605Sergey Koren A src/AS_RUN/fragmentDepth.C jcvi20081008Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20081002Sergey Koren A src/AS_RUN/fragmentDepth.C jcvi20080627Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20071128Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20071128Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20071108Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20070904Sergey Koren A src/AS_RUN/fragmentDepth.C jcvi20070427Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20070403Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20070403Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20070329Brian P. Walenz A src/AS_RUN/fragmentDepth.C jcvi20070328Brian P. Walenz A src/AS_RUN/replaceUIDwithName-posmap.pl nihh20151012Brian P. Walenz A src/AS_RUN/replaceUIDwithName-posmap.pl bnbi20141001Brian P. Walenz A src/AS_RUN/replaceUIDwithName-posmap.pl bnbi20141001Brian P. Walenz A src/AS_RUN/replaceUIDwithName-posmap.pl bnbi20141001Brian P. Walenz A src/AS_RUN/replaceUIDwithName-posmap.pl jcvi20130922Brian P. Walenz D src/AS_RUN/replaceUIDwithName-posmap.pl src/AS_RUN/replaceUIDwithName-simple.pl A src/falcon_sense/outputFalcon.H bnbi20150420Brian P. Walenz A src/falcon_sense/outputFalcon.C bnbi20150520Brian P. Walenz A src/falcon_sense/outputFalcon.C bnbi20150420Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.C bnbi20150814Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.C bnbi20150701Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.C bnbi20150603Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.C bnbi20150514Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.C bnbi20150420Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.C bnbi20150415Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.C bnbi20150409Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.C bnbi20150409Brian P. Walenz A src/fastq-utilities/fastqSample.C bnbi20150224Brian P. Walenz A src/fastq-utilities/fastqSample.C bnbi20150113Brian P. Walenz A src/fastq-utilities/fastqSample.C bnbi20141001Brian P. Walenz A src/fastq-utilities/fastqSample.C bnbi20140806Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20130801Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20130801Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20121023Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20120529Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20111229Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20110819Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20110728Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20110719Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20110719Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20110104Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20100402Brian P. Walenz A src/fastq-utilities/fastqSample.C jcvi20100222Brian P. Walenz D src/fastq-utilities/fastqSample.C src/AS_GKP/fastqSample.C A src/fastq-utilities/fastqSimulate-sort.C bnbi20150113Brian P. Walenz A src/fastq-utilities/fastqSimulate-sort.C jcvi20130801Brian P. Walenz A src/fastq-utilities/fastqSimulate-sort.C jcvi20130801Brian P. Walenz A src/fastq-utilities/fastqSimulate-sort.C jcvi20130321Brian P. Walenz D src/fastq-utilities/fastqSimulate-sort.C src/AS_GKP/fastqSimulate-sort.C A src/fastq-utilities/fastqSimulate-perfectSep.pl nihh20151012Brian P. Walenz A src/fastq-utilities/fastqSimulate-perfectSep.pl bnbi20150113Brian P. Walenz A src/fastq-utilities/fastqSimulate-perfectSep.pl jcvi20130801Brian P. Walenz A src/fastq-utilities/fastqSimulate-perfectSep.pl jcvi20111228Brian P. Walenz D src/fastq-utilities/fastqSimulate-perfectSep.pl src/AS_GKP/fastqSimulate-perfectSep.pl A src/fastq-utilities/fastqSimulate.C bnbi20150204Brian P. Walenz A src/fastq-utilities/fastqSimulate.C bnbi20150202Brian P. Walenz A src/fastq-utilities/fastqSimulate.C bnbi20150131Brian P. Walenz A src/fastq-utilities/fastqSimulate.C bnbi20150131Brian P. Walenz A src/fastq-utilities/fastqSimulate.C bnbi20150113Brian P. Walenz A src/fastq-utilities/fastqSimulate.C bnbi20140806Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20140331Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20140304Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20140220Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20140129Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20130801Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20130801Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20130318Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20120709Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20120313Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20120118Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110907Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110906Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110829Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110815Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110803Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110728Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110705Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110627Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110623Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110623Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110603Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110421Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110224Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110222Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110222Brian P. Walenz A src/fastq-utilities/fastqSimulate.C jcvi20110221Brian P. Walenz D src/fastq-utilities/fastqSimulate.C src/AS_GKP/fastqSimulate.C A src/fastq-utilities/fastqSimulate-checkCoverage.pl nihh20151012Brian P. Walenz A src/fastq-utilities/fastqSimulate-checkCoverage.pl bnbi20150113Brian P. Walenz A src/fastq-utilities/fastqSimulate-checkCoverage.pl jcvi20140220Brian P. Walenz D src/fastq-utilities/fastqSimulate-checkCoverage.pl src/AS_GKP/fastqSimulate-checkCoverage.pl A src/fastq-utilities/fastqAnalyze.C bnbi20150113Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20130801Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20130801Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20130215Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20130215Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20130107Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20121213Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20121212Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20120625Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20120321Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20120225Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C jcvi20120224Brian P. Walenz D src/fastq-utilities/fastqAnalyze.C src/AS_GKP/fastqAnalyze.C A src/AS_UTL/bitOperations.H bnbi20141208Brian P. Walenz A src/AS_UTL/bitOperations.H bnbi20141205Brian P. Walenz A src/AS_UTL/bitOperations.H jcvi20140411Brian P. Walenz A src/AS_UTL/bitOperations.H jcvi20080707Brian P. Walenz A src/AS_UTL/bitOperations.H jcvi20060728Brian P. Walenz A src/AS_UTL/bitOperations.H jcvi20050310Brian P. Walenz A src/AS_UTL/bitOperations.H jcvi20050223Brian P. Walenz A src/AS_UTL/bitOperations.H none20040427Brian P. Walenz D src/AS_UTL/bitOperations.H kmer/libutil/bitOperations.h A src/AS_UTL/intervalList.H nihh20151030Brian P. Walenz A src/AS_UTL/intervalList.H bnbi20141205Brian P. Walenz A src/AS_UTL/intervalList.H bnbi20141022Brian P. Walenz A src/AS_UTL/intervalList.H bnbi20141021Brian P. Walenz A src/AS_UTL/intervalList.H bnbi20141014Brian P. Walenz A src/AS_UTL/intervalList.H bnbi20141007Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20140411Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20130508Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20130429Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20130404Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20101201Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20070608Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20061212Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20061023Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20060825Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20060808Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20060407Brian P. Walenz A src/AS_UTL/intervalList.H jcvi20050712Brian P. Walenz A src/AS_UTL/intervalList.H none20040421Brian P. Walenz A src/AS_UTL/intervalList.H craa20030813Brian P. Walenz A src/AS_UTL/intervalList.H craa20030811Brian P. Walenz D src/AS_UTL/intervalList.H kmer/libutil/intervalList.H A src/AS_UTL/AS_UTL_reverseComplement.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.H jcvi20081008Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.H jcvi20080627Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.H jcvi20080515Brian P. Walenz A src/AS_UTL/sweatShop.C bnbi20150624Brian P. Walenz A src/AS_UTL/sweatShop.C bnbi20141208Brian P. Walenz A src/AS_UTL/sweatShop.C bnbi20141205Brian P. Walenz A src/AS_UTL/sweatShop.C bnbi20141205Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20140411Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20130103Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20121015Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20110816Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20110119Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20110110Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20100919Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20100331Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20081001Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20080925Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20080904Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20080903Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20080902Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20080828Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20080820Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20080819Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20080818Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20080818Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20080814Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20060512Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20060501Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20060407Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20060308Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20060303Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20060302Brian P. Walenz A src/AS_UTL/sweatShop.C jcvi20060302Brian P. Walenz D src/AS_UTL/sweatShop.C kmer/libutil/sweatShop.C A src/AS_UTL/decodeBooleanString.C bnbi20141126Brian P. Walenz A src/AS_UTL/readBuffer.H bnbi20141208Brian P. Walenz A src/AS_UTL/readBuffer.H jcvi20140411Brian P. Walenz A src/AS_UTL/readBuffer.H jcvi20101112Brian P. Walenz A src/AS_UTL/readBuffer.H jcvi20100919Brian P. Walenz A src/AS_UTL/readBuffer.H jcvi20080911Brian P. Walenz A src/AS_UTL/readBuffer.H jcvi20080909Brian P. Walenz A src/AS_UTL/readBuffer.H jcvi20071226Brian P. Walenz A src/AS_UTL/readBuffer.H jcvi20070320Brian P. Walenz A src/AS_UTL/readBuffer.H jcvi20060624Brian P. Walenz A src/AS_UTL/readBuffer.H none20040524Brian P. Walenz A src/AS_UTL/readBuffer.H none20040421Brian P. Walenz A src/AS_UTL/readBuffer.H none20040329Brian P. Walenz A src/AS_UTL/readBuffer.H none20040317Brian P. Walenz A src/AS_UTL/readBuffer.H craa20040106Brian P. Walenz A src/AS_UTL/readBuffer.H craa20030929Brian P. Walenz A src/AS_UTL/readBuffer.H craa20030915Brian P. Walenz A src/AS_UTL/readBuffer.H craa20030708Brian P. Walenz A src/AS_UTL/readBuffer.H craa20030612Brian P. Walenz A src/AS_UTL/readBuffer.H craa20030506Brian P. Walenz A src/AS_UTL/readBuffer.H craa20030416Brian P. Walenz A src/AS_UTL/readBuffer.H craa20030415Brian P. Walenz A src/AS_UTL/readBuffer.H craa20030415Brian P. Walenz D src/AS_UTL/readBuffer.H kmer/libutil/readBuffer.H A src/AS_UTL/AS_UTL_fileIO.C bnbi20150814Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C bnbi20150514Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C bnbi20150325Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20130927Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20130922Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20121114Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20120402Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20120306Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20111229Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20091202Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20091107Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20090610Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20090511Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20081008Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20080627Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070929Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070902Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070827Eli Venter A src/AS_UTL/AS_UTL_fileIO.C jcvi20070824Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070723Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070723Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070603Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070416Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070402Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070313Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070301Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070228Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20070218Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20061008Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20050928Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C jcvi20050926Brian P. Walenz A src/AS_UTL/kMerTiny.H nihh20151029Brian P. Walenz A src/AS_UTL/kMerTiny.H bnbi20141205Brian P. Walenz A src/AS_UTL/kMerTiny.H bnbi20141205Brian P. Walenz A src/AS_UTL/kMerTiny.H bnbi20141205Brian P. Walenz A src/AS_UTL/kMerTiny.H jcvi20140411Brian P. Walenz A src/AS_UTL/kMerTiny.H jcvi20100915Brian P. Walenz A src/AS_UTL/kMerTiny.H jcvi20080808Brian P. Walenz A src/AS_UTL/kMerTiny.H jcvi20080728Brian P. Walenz A src/AS_UTL/kMerTiny.H jcvi20080609Brian P. Walenz A src/AS_UTL/kMerTiny.H jcvi20080318Brian P. Walenz A src/AS_UTL/kMerTiny.H jcvi20071208Brian P. Walenz A src/AS_UTL/kMerTiny.H jcvi20071023Brian P. Walenz A src/AS_UTL/kMerTiny.H jcvi20070913Brian P. Walenz A src/AS_UTL/kMerTiny.H jcvi20070913Brian P. Walenz D src/AS_UTL/kMerTiny.H kmer/libbio/kmertiny.H A src/AS_UTL/bitPackedFile.H bnbi20141205Brian P. Walenz A src/AS_UTL/bitPackedFile.H bnbi20141205Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20140411Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20080708Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20070827Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20070823Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20070320Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20070102Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20061026Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20061022Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20060706Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20060621Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20051127Brian P. Walenz A src/AS_UTL/bitPackedFile.H jcvi20050712Brian P. Walenz A src/AS_UTL/bitPackedFile.H none20040421Brian P. Walenz A src/AS_UTL/bitPackedFile.H none20040331Brian P. Walenz A src/AS_UTL/bitPackedFile.H none20040329Brian P. Walenz A src/AS_UTL/bitPackedFile.H craa20031020Brian P. Walenz A src/AS_UTL/bitPackedFile.H craa20030909Brian P. Walenz A src/AS_UTL/bitPackedFile.H craa20030506Brian P. Walenz A src/AS_UTL/bitPackedFile.H craa20030220Brian P. Walenz A src/AS_UTL/bitPackedFile.H craa20030102Brian P. Walenz D src/AS_UTL/bitPackedFile.H kmer/libutil/bitPackedFile.H A src/AS_UTL/speedCounter.C bnbi20141208Brian P. Walenz A src/AS_UTL/speedCounter.C bnbi20141205Brian P. Walenz A src/AS_UTL/speedCounter.C jcvi20140411Brian P. Walenz A src/AS_UTL/speedCounter.C jcvi20061022Brian P. Walenz D src/AS_UTL/speedCounter.C kmer/libutil/speedCounter.C A src/AS_UTL/testRand.C nihh20151012Brian P. Walenz A src/AS_UTL/testRand.C jcvi20130801Brian P. Walenz A src/AS_UTL/testRand.C jcvi20130801Brian P. Walenz A src/AS_UTL/testRand.C jcvi20130801Brian P. Walenz A src/AS_UTL/testRand.C jcvi20081008Brian P. Walenz A src/AS_UTL/testRand.C jcvi20080627Brian P. Walenz A src/AS_UTL/bitEncodings.C bnbi20141208Brian P. Walenz A src/AS_UTL/stddev.H nihh20151027Brian P. Walenz A src/AS_UTL/stddev.H nihh20151027Brian P. Walenz A src/AS_UTL/stddev.H bnbi20150818Brian P. Walenz A src/AS_UTL/stddev.H bnbi20150726Brian P. Walenz A src/AS_UTL/stddev.H bnbi20150723Brian P. Walenz A src/AS_UTL/dnaAlphabets.C bnbi20141208Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.C bnbi20150528Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.C jcvi20131011Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.C jcvi20120213Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.C jcvi20120212Brian P. Walenz A src/AS_UTL/AS_UTL_stackTraceTest.C jcvi20131018Brian P. Walenz A src/AS_UTL/bitPackedArray.H bnbi20141208Brian P. Walenz A src/AS_UTL/bitPackedArray.H bnbi20141205Brian P. Walenz A src/AS_UTL/bitPackedArray.H bnbi20141205Brian P. Walenz A src/AS_UTL/bitPackedArray.H jcvi20140411Brian P. Walenz A src/AS_UTL/bitPackedArray.H jcvi20061026Brian P. Walenz A src/AS_UTL/bitPackedArray.H jcvi20060706Brian P. Walenz A src/AS_UTL/bitPackedArray.H jcvi20060621Brian P. Walenz A src/AS_UTL/bitPackedArray.H jcvi20050712Brian P. Walenz D src/AS_UTL/bitPackedArray.H kmer/libutil/bitPackedArray.H A src/AS_UTL/AS_UTL_alloc.C nihh20151008Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20120508Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20110719Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20081008Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20080627Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20070830Eli Venter A src/AS_UTL/AS_UTL_alloc.C jcvi20070218Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20070215Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20070214Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20070213Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20061106Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20061008Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20060926Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20050825Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20050825Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C jcvi20050824Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.H jcvi20130314Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.H jcvi20120508Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.H jcvi20100322Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.H jcvi20090331Sergey Koren A src/AS_UTL/AS_UTL_fasta.H jcvi20081029Sergey Koren A src/AS_UTL/AS_UTL_fasta.H jcvi20081008Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.H jcvi20080627Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.H jcvi20080515Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.H jcvi20071102Brian P. Walenz A src/AS_UTL/timeAndSize.C bnbi20141208Brian P. Walenz A src/AS_UTL/kMer.C nihh20151029Brian P. Walenz A src/AS_UTL/kMer.C bnbi20141205Brian P. Walenz A src/AS_UTL/kMer.C bnbi20141205Brian P. Walenz A src/AS_UTL/kMer.C bnbi20141205Brian P. Walenz A src/AS_UTL/kMer.C jcvi20140411Brian P. Walenz A src/AS_UTL/kMer.C jcvi20111214Brian P. Walenz A src/AS_UTL/kMer.C jcvi20110330Brian P. Walenz D src/AS_UTL/kMer.C kmer/libbio/kmer.C A src/AS_UTL/bitPackedFile.C nihh20151029Brian P. Walenz A src/AS_UTL/bitPackedFile.C bnbi20141208Brian P. Walenz A src/AS_UTL/bitPackedFile.C bnbi20141205Brian P. Walenz A src/AS_UTL/bitPackedFile.C bnbi20141205Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20140411Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20120227Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20080708Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20070827Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20070102Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20061026Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20061022Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20060706Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20060621Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20051127Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20050712Brian P. Walenz A src/AS_UTL/bitPackedFile.C jcvi20050316Brian P. Walenz A src/AS_UTL/bitPackedFile.C none20041010Brian P. Walenz A src/AS_UTL/bitPackedFile.C none20040514Brian P. Walenz A src/AS_UTL/bitPackedFile.C none20040430Brian P. Walenz A src/AS_UTL/bitPackedFile.C none20040427Brian P. Walenz A src/AS_UTL/bitPackedFile.C none20040421Brian P. Walenz A src/AS_UTL/bitPackedFile.C craa20040401Brian P. Walenz A src/AS_UTL/bitPackedFile.C none20040331Brian P. Walenz A src/AS_UTL/bitPackedFile.C none20040329Brian P. Walenz A src/AS_UTL/bitPackedFile.C craa20031020Brian P. Walenz A src/AS_UTL/bitPackedFile.C craa20031009Brian P. Walenz A src/AS_UTL/bitPackedFile.C craa20030909Brian P. Walenz A src/AS_UTL/bitPackedFile.C craa20030506Brian P. Walenz A src/AS_UTL/bitPackedFile.C craa20030220Brian P. Walenz A src/AS_UTL/bitPackedFile.C craa20030131Brian P. Walenz A src/AS_UTL/bitPackedFile.C craa20030102Brian P. Walenz D src/AS_UTL/bitPackedFile.C kmer/libutil/bitPackedFile.C A src/AS_UTL/speedCounter.H bnbi20141208Brian P. Walenz A src/AS_UTL/speedCounter.H bnbi20141205Brian P. Walenz A src/AS_UTL/speedCounter.H jcvi20140411Brian P. Walenz A src/AS_UTL/speedCounter.H jcvi20130103Brian P. Walenz A src/AS_UTL/speedCounter.H jcvi20120229Brian P. Walenz A src/AS_UTL/speedCounter.H jcvi20061012Brian P. Walenz A src/AS_UTL/speedCounter.H jcvi20050712Brian P. Walenz A src/AS_UTL/speedCounter.H craa20030506Brian P. Walenz A src/AS_UTL/speedCounter.H craa20030102Brian P. Walenz D src/AS_UTL/speedCounter.H kmer/libutil/speedCounter.H A src/AS_UTL/bitEncodings.H bnbi20141208Brian P. Walenz A src/AS_UTL/memoryMappedFile.H bnbi20150421Brian P. Walenz A src/AS_UTL/memoryMappedFile.H bnbi20150306Brian P. Walenz A src/AS_UTL/memoryMappedFile.H bnbi20150211Brian P. Walenz A src/AS_UTL/memoryMappedFile.H bnbi20150204Brian P. Walenz A src/AS_UTL/memoryMappedFile.H bnbi20141223Brian P. Walenz A src/AS_UTL/memoryMappedFile.H bnbi20141209Brian P. Walenz A src/AS_UTL/memoryMappedFile.H bnbi20141208Brian P. Walenz A src/AS_UTL/memoryMappedFile.H bnbi20141127Brian P. Walenz A src/AS_UTL/memoryMappedFile.H bnbi20141126Brian P. Walenz A src/AS_UTL/memoryMappedFile.H jcvi20130801Brian P. Walenz A src/AS_UTL/memoryMappedFile.H jcvi20120219Brian P. Walenz A src/AS_UTL/memoryMappedFile.H jcvi20120218Brian P. Walenz A src/AS_UTL/memoryMappedFile.H jcvi20120216Brian P. Walenz D src/AS_UTL/memoryMappedFile.H src/AS_BAT/memoryMappedFile.H A src/AS_UTL/AS_UTL_reverseComplement.C bnbi20141205Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.C jcvi20090529Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.C jcvi20081008Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.C jcvi20080627Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.C jcvi20080515Brian P. Walenz A src/AS_UTL/sweatShop.H bnbi20141208Brian P. Walenz A src/AS_UTL/sweatShop.H bnbi20141205Brian P. Walenz A src/AS_UTL/sweatShop.H jcvi20140411Brian P. Walenz A src/AS_UTL/sweatShop.H jcvi20110816Brian P. Walenz A src/AS_UTL/sweatShop.H jcvi20101102Brian P. Walenz A src/AS_UTL/sweatShop.H jcvi20100331Brian P. Walenz A src/AS_UTL/sweatShop.H jcvi20080904Brian P. Walenz A src/AS_UTL/sweatShop.H jcvi20080902Brian P. Walenz A src/AS_UTL/sweatShop.H jcvi20080820Brian P. Walenz A src/AS_UTL/sweatShop.H jcvi20060501Brian P. Walenz A src/AS_UTL/sweatShop.H jcvi20060308Brian P. Walenz A src/AS_UTL/sweatShop.H jcvi20060302Brian P. Walenz D src/AS_UTL/sweatShop.H kmer/libutil/sweatShop.H A src/AS_UTL/readBuffer.C nihh20151029Brian P. Walenz A src/AS_UTL/readBuffer.C bnbi20141208Brian P. Walenz A src/AS_UTL/readBuffer.C bnbi20140822Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20140411Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20101112Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20100919Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20090724Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20080925Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20080912Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20080911Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20080909Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20080129Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20071226Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20060914Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20060620Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20050712Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20050523Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20050519Brian P. Walenz A src/AS_UTL/readBuffer.C jcvi20050401Brian P. Walenz A src/AS_UTL/readBuffer.C none20041010Brian P. Walenz A src/AS_UTL/readBuffer.C none20040511Brian P. Walenz A src/AS_UTL/readBuffer.C craa20040506Brian P. Walenz D src/AS_UTL/readBuffer.C kmer/libutil/readBuffer.C A src/AS_UTL/AS_UTL_fileIO.H bnbi20150814Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H bnbi20141208Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H bnbi20141205Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20130927Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20130922Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20120508Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20091202Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20091107Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20090610Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20081008Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20080627Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20070603Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20070416Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20070228Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20070218Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H jcvi20050926Brian P. Walenz A src/AS_UTL/splitToWords.H bnbi20150811Brian P. Walenz A src/AS_UTL/splitToWords.H bnbi20141205Brian P. Walenz A src/AS_UTL/splitToWords.H jcvi20140411Brian P. Walenz A src/AS_UTL/splitToWords.H jcvi20120718Brian P. Walenz A src/AS_UTL/splitToWords.H jcvi20060718Brian P. Walenz A src/AS_UTL/splitToWords.H jcvi20050712Brian P. Walenz D src/AS_UTL/splitToWords.H kmer/libutil/splitToWords.H A src/AS_UTL/findKeyAndValue.H bnbi20150130Brian P. Walenz A src/AS_UTL/findKeyAndValue.H bnbi20141202Brian P. Walenz A src/AS_UTL/findKeyAndValue.H bnbi20141127Brian P. Walenz A src/AS_UTL/findKeyAndValue.H bnbi20141121Brian P. Walenz A src/AS_UTL/kMer.H bnbi20141205Brian P. Walenz A src/AS_UTL/kMer.H bnbi20141205Brian P. Walenz A src/AS_UTL/kMer.H bnbi20141205Brian P. Walenz A src/AS_UTL/kMer.H jcvi20140411Brian P. Walenz A src/AS_UTL/kMer.H jcvi20110330Brian P. Walenz A src/AS_UTL/kMer.H jcvi20100831Brian P. Walenz A src/AS_UTL/kMer.H jcvi20100225Brian P. Walenz A src/AS_UTL/kMer.H jcvi20081008Brian P. Walenz A src/AS_UTL/kMer.H jcvi20080808Brian P. Walenz A src/AS_UTL/kMer.H jcvi20080728Brian P. Walenz A src/AS_UTL/kMer.H jcvi20080318Brian P. Walenz A src/AS_UTL/kMer.H jcvi20080318Brian P. Walenz A src/AS_UTL/kMer.H jcvi20071028Brian P. Walenz A src/AS_UTL/kMer.H jcvi20071011Brian P. Walenz A src/AS_UTL/kMer.H jcvi20071009Brian P. Walenz A src/AS_UTL/kMer.H jcvi20071009Brian P. Walenz A src/AS_UTL/kMer.H jcvi20070926Brian P. Walenz A src/AS_UTL/kMer.H jcvi20070913Brian P. Walenz A src/AS_UTL/kMer.H jcvi20070913Brian P. Walenz A src/AS_UTL/kMer.H jcvi20070326Brian P. Walenz A src/AS_UTL/kMer.H jcvi20060621Brian P. Walenz A src/AS_UTL/kMer.H jcvi20060512Brian P. Walenz A src/AS_UTL/kMer.H jcvi20051127Brian P. Walenz A src/AS_UTL/kMer.H jcvi20051012Brian P. Walenz A src/AS_UTL/kMer.H jcvi20050910Brian P. Walenz A src/AS_UTL/kMer.H jcvi20050712Brian P. Walenz A src/AS_UTL/kMer.H jcvi20050531Brian P. Walenz A src/AS_UTL/kMer.H jcvi20050531Brian P. Walenz A src/AS_UTL/kMer.H jcvi20050523Brian P. Walenz A src/AS_UTL/kMer.H jcvi20050519Brian P. Walenz D src/AS_UTL/kMer.H kmer/libbio/kmer.H A src/AS_UTL/bitPacking.H bnbi20141205Brian P. Walenz A src/AS_UTL/bitPacking.H jcvi20140411Brian P. Walenz A src/AS_UTL/bitPacking.H jcvi20080912Brian P. Walenz A src/AS_UTL/bitPacking.H jcvi20080707Brian P. Walenz A src/AS_UTL/bitPacking.H jcvi20071211Brian P. Walenz A src/AS_UTL/bitPacking.H jcvi20071205Brian P. Walenz A src/AS_UTL/bitPacking.H none20040623Brian P. Walenz A src/AS_UTL/bitPacking.H none20040622Brian P. Walenz A src/AS_UTL/bitPacking.H craa20040506Brian P. Walenz A src/AS_UTL/bitPacking.H none20040427Brian P. Walenz D src/AS_UTL/bitPacking.H kmer/libutil/bitPacking.h A src/AS_UTL/timeAndSize.H bnbi20141208Brian P. Walenz A src/AS_UTL/bitPackedArray.C bnbi20141208Brian P. Walenz A src/AS_UTL/bitPackedArray.C bnbi20141205Brian P. Walenz A src/AS_UTL/bitPackedArray.C bnbi20141205Brian P. Walenz A src/AS_UTL/bitPackedArray.C jcvi20140411Brian P. Walenz A src/AS_UTL/bitPackedArray.C jcvi20121015Brian P. Walenz A src/AS_UTL/bitPackedArray.C jcvi20061026Brian P. Walenz A src/AS_UTL/bitPackedArray.C jcvi20060621Brian P. Walenz A src/AS_UTL/bitPackedArray.C jcvi20050207Brian P. Walenz D src/AS_UTL/bitPackedArray.C kmer/libutil/bitPackedArray.C A src/AS_UTL/AS_UTL_alloc.H nihh20151029Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H nihh20151008Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H bnbi20150131Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H bnbi20150113Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H bnbi20150112Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H bnbi20150107Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H bnbi20141222Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H bnbi20140811Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H jcvi20081008Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H jcvi20080627Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H jcvi20070214Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H jcvi20070214Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H jcvi20061106Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H jcvi20060926Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H jcvi20050824Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.C jcvi20130417Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.C jcvi20130329Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.C jcvi20130320Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.C jcvi20130314Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C bnbi20150227Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C bnbi20130320Sergey Koren A src/AS_UTL/AS_UTL_fasta.C jcvi20120508Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C jcvi20100322Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C jcvi20090331Sergey Koren A src/AS_UTL/AS_UTL_fasta.C jcvi20081029Sergey Koren A src/AS_UTL/AS_UTL_fasta.C jcvi20081008Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C jcvi20080627Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C jcvi20080604Sergey Koren A src/AS_UTL/AS_UTL_fasta.C jcvi20080515Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C jcvi20071102Brian P. Walenz A src/AS_UTL/stddev.C bnbi20150818Brian P. Walenz A src/AS_UTL/stddev.C bnbi20150726Brian P. Walenz A src/AS_UTL/stddev.C bnbi20150723Brian P. Walenz A src/AS_UTL/stddev.C bnbi20150410Brian P. Walenz A src/AS_UTL/stddev.C jcvi20130923Brian P. Walenz A src/AS_UTL/stddev.C jcvi20130801Brian P. Walenz A src/AS_UTL/stddev.C jcvi20121218Brian P. Walenz A src/AS_UTL/stddev.C jcvi20121213Brian P. Walenz A src/AS_UTL/stddev.C jcvi20121205Brian P. Walenz A src/AS_UTL/stddev.C jcvi20121204Brian P. Walenz D src/AS_UTL/stddev.C src/AS_TER/analyzePosMap-libraryFate.C A src/AS_UTL/dnaAlphabets.H bnbi20141205Brian P. Walenz A src/AS_UTL/dnaAlphabets.H bnbi20141205Brian P. Walenz A src/AS_UTL/dnaAlphabets.H bnbi20141205Brian P. Walenz A src/AS_UTL/dnaAlphabets.H bnbi20141205Brian P. Walenz A src/AS_UTL/dnaAlphabets.H jcvi20080728Brian P. Walenz A src/AS_UTL/dnaAlphabets.H jcvi20080303Brian P. Walenz A src/AS_UTL/dnaAlphabets.H jcvi20071208Brian P. Walenz A src/AS_UTL/dnaAlphabets.H jcvi20060407Brian P. Walenz A src/AS_UTL/dnaAlphabets.H jcvi20050305Brian P. Walenz A src/AS_UTL/dnaAlphabets.H none20040528Brian P. Walenz A src/AS_UTL/dnaAlphabets.H craa20031021Brian P. Walenz A src/AS_UTL/dnaAlphabets.H craa20031016Brian P. Walenz A src/AS_UTL/dnaAlphabets.H craa20030506Brian P. Walenz D src/AS_UTL/dnaAlphabets.H kmer/libbio/alphabet-generate.c D src/AS_UTL/dnaAlphabets.H kmer/libbio/dnaAlphabets.H A src/AS_UTL/kMerHuge.H bnbi20141205Brian P. Walenz A src/AS_UTL/kMerHuge.H bnbi20141205Brian P. Walenz A src/AS_UTL/kMerHuge.H bnbi20141205Brian P. Walenz A src/AS_UTL/kMerHuge.H jcvi20140411Brian P. Walenz A src/AS_UTL/kMerHuge.H jcvi20080808Brian P. Walenz A src/AS_UTL/kMerHuge.H jcvi20080609Brian P. Walenz A src/AS_UTL/kMerHuge.H jcvi20080318Brian P. Walenz A src/AS_UTL/kMerHuge.H jcvi20071208Brian P. Walenz A src/AS_UTL/kMerHuge.H jcvi20071023Brian P. Walenz A src/AS_UTL/kMerHuge.H jcvi20070913Brian P. Walenz D src/AS_UTL/kMerHuge.H kmer/libbio/kmerhuge.H A src/AS_UTL/AS_UTL_decodeRange.H bnbi20150528Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.H jcvi20131011Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.H jcvi20130801Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.H jcvi20120220Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.H jcvi20120212Brian P. Walenz A src/AS_UTL/memoryMappedFileTest.C bnbi20141127Brian P. Walenz A src/AS_UTL/memoryMappedFileTest.C bnbi20141126Brian P. Walenz D src/AS_UTL/memoryMappedFileTest.C src/AS_BAT/memoryMappedFileTest.C A src/correction/readConsensus.C nihh20151029Brian P. Walenz A src/correction/readConsensus.C bnbi20150726Brian P. Walenz A src/correction/readConsensus.C bnbi20150720Brian P. Walenz A src/correction/readConsensus.C bnbi20150708Brian P. Walenz A src/correction/readConsensus.C bnbi20150701Brian P. Walenz A src/correction/readConsensus.C bnbi20150701Brian P. Walenz A src/correction/readConsensus.C bnbi20150624Brian P. Walenz A src/correction/readConsensus.C bnbi20150623Brian P. Walenz A src/correction/filterCorrectionOverlaps.C nihh20151029Brian P. Walenz A src/correction/filterCorrectionOverlaps.C bnbi20150625Brian P. Walenz A src/correction/filterCorrectionOverlaps.C bnbi20150616Brian P. Walenz A src/correction/filterCorrectionOverlaps.C bnbi20150616Brian P. Walenz A src/correction/filterCorrectionOverlaps.C bnbi20150615Brian P. Walenz A src/correction/filterCorrectionOverlaps.C bnbi20150605Brian P. Walenz A src/correction/filterCorrectionOverlaps.C bnbi20150528Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150921Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150625Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150616Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150616Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150615Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150612Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150610Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150605Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150603Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150528Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150526Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150520Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150514Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150509Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150420Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150415Brian P. Walenz A src/correction/generateCorrectionLayouts.C bnbi20150409Brian P. Walenz A src/overlapBasedTrimming/generate-random-chimeric-fragments.pl nihh20151012Brian P. Walenz A src/overlapBasedTrimming/trimReads.C bnbi20150625Brian P. Walenz A src/overlapBasedTrimming/trimReads.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/trimReads.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/trimReads.C bnbi20150603Brian P. Walenz A src/overlapBasedTrimming/trimReads.C bnbi20150528Brian P. Walenz A src/overlapBasedTrimming/trimReads.C bnbi20150514Brian P. Walenz A src/overlapBasedTrimming/trimReads.C bnbi20150420Brian P. Walenz A src/overlapBasedTrimming/trimReads.C bnbi20150415Brian P. Walenz A src/overlapBasedTrimming/finalTrim-stats.pl nihh20151012Brian P. Walenz A src/overlapBasedTrimming/splitReads.H bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/splitReads.H bnbi20150420Brian P. Walenz A src/overlapBasedTrimming/splitReads-workUnit.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/splitReads-workUnit.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/splitReads-workUnit.C bnbi20150420Brian P. Walenz A src/overlapBasedTrimming/splitReads-workUnit.C bnbi20150415Brian P. Walenz A src/overlapBasedTrimming/adjustOverlaps.H bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/detect-unsplit-subreads-in-overlaps.pl nihh20151012Brian P. Walenz A src/overlapBasedTrimming/trimReads-bestEdge.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/trimReads-bestEdge.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/trimReads-bestEdge.C bnbi20150528Brian P. Walenz A src/overlapBasedTrimming/trimReads-largestCovered.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/trimReads-largestCovered.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/trimReads-largestCovered.C bnbi20150528Brian P. Walenz A src/overlapBasedTrimming/test-random-chimeric-fragments.pl nihh20151012Brian P. Walenz A src/overlapBasedTrimming/adjustFlipped.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/adjustFlipped.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/adjustFlipped.C bnbi20150415Brian P. Walenz A src/overlapBasedTrimming/splitReads.C bnbi20150625Brian P. Walenz A src/overlapBasedTrimming/splitReads.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/splitReads.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/splitReads.C bnbi20150420Brian P. Walenz A src/overlapBasedTrimming/splitReads.C bnbi20150415Brian P. Walenz A src/overlapBasedTrimming/trimReads.H bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/trimReads.H bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/adjustNormal.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/adjustNormal.C bnbi20150616Brian P. Walenz A src/overlapBasedTrimming/adjustNormal.C bnbi20150415Brian P. Walenz A src/overlapBasedTrimming/clearRangeFile.H bnbi20150415Brian P. Walenz A src/erateEstimate/erate-estimate-plot-per-base-estimate.pl nihh20151012Brian P. Walenz A src/erateEstimate/erate-estimate-plot-per-base-estimate.pl bnbi20150807Brian P. Walenz A src/erateEstimate/erate-estimate-plot-per-base-estimate.pl bnbi20141115Brian P. Walenz D src/erateEstimate/erate-estimate-plot-per-base-estimate.pl src/AS_BAT/erate-estimate-plot-per-base-estimate.pl A src/erateEstimate/erate-estimate-test-based-on-mapping.pl nihh20151012Brian P. Walenz A src/erateEstimate/erate-estimate-test-based-on-mapping.pl bnbi20150807Brian P. Walenz A src/erateEstimate/erate-estimate-test-based-on-mapping.pl bnbi20141115Brian P. Walenz D src/erateEstimate/erate-estimate-test-based-on-mapping.pl src/AS_BAT/erate-estimate-test-based-on-mapping.pl A src/erateEstimate/erateEstimate.C bnbi20150729Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20150701Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20150625Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20150616Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20150616Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20150529Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20150514Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20150317Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20150303Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20150227Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20141223Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20141223Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20141223Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20141219Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20141117Brian P. Walenz A src/erateEstimate/erateEstimate.C bnbi20141021Brian P. Walenz D src/erateEstimate/erateEstimate.C src/AS_BAT/erate-estimate.C A src/AS_UTL/md5.c jcvi20140411Brian P. Walenz A src/AS_UTL/md5.c jcvi20080724Brian P. Walenz A src/AS_UTL/md5.c jcvi20050701Brian P. Walenz A src/AS_UTL/md5.c jcvi20050302Brian P. Walenz A src/AS_UTL/md5.c none20041010Brian P. Walenz A src/AS_UTL/md5.c craa20040430Brian P. Walenz A src/AS_UTL/md5.c craa20040421Brian P. Walenz A src/AS_UTL/md5.c craa20040220Brian P. Walenz A src/AS_UTL/md5.c craa20030729Brian P. Walenz A src/AS_UTL/md5.c craa20030721Brian P. Walenz A src/AS_UTL/md5.c jcvi20140411Brian P. Walenz A src/AS_UTL/md5.c jcvi20080724Brian P. Walenz A src/AS_UTL/md5.c jcvi20050712Brian P. Walenz A src/AS_UTL/md5.c none20040302Brian P. Walenz A src/AS_UTL/md5.c none20041010Brian P. Walenz A src/AS_UTL/md5.c craa20040430Brian P. Walenz A src/AS_UTL/md5.c craa20040421Brian P. Walenz A src/AS_UTL/md5.c craa20040220Brian P. Walenz A src/AS_UTL/md5.c craa20030721Brian P. Walenz D src/AS_UTL/md5.H kmer/libutil/md5.h D src/AS_UTL/md5.C kmer/libutil/md5.c A documentation/source/quick-start.rst nihh20160303Sergey Koren A documentation/source/commands/updateDocs.sh nihh20160303Sergey Koren A documentation/source/command-reference.rst nihh20160303Sergey Koren A documentation/source/commands/bogart.rst nihh20160303Sergey Koren A documentation/source/commands/bogus.rst nihh20160303Sergey Koren A documentation/source/commands/canu.rst nihh20160303Sergey Koren A documentation/source/commands/correctOverlaps.rst nihh20160303Sergey Koren A documentation/source/commands/createFalconSenseInputs.rst nihh20160303Sergey Koren A documentation/source/commands/erateEstimate.rst nihh20160303Sergey Koren A documentation/source/commands/estimate-mer-threshold.rst nihh20160303Sergey Koren A documentation/source/commands/fastqAnalyze.rst nihh20160303Sergey Koren A documentation/source/commands/fastqSample.rst nihh20160303Sergey Koren A documentation/source/commands/fastqSimulate-sort.rst nihh20160303Sergey Koren A documentation/source/commands/fastqSimulate.rst nihh20160303Sergey Koren A documentation/source/commands/filterCorrectionOverlaps.rst nihh20160303Sergey Koren A documentation/source/commands/findErrors.rst nihh20160303Sergey Koren A documentation/source/commands/gatekeeperCreate.rst nihh20160303Sergey Koren A documentation/source/commands/gatekeeperDumpFASTQ.rst nihh20160303Sergey Koren A documentation/source/commands/gatekeeperDumpMetaData.rst nihh20160303Sergey Koren A documentation/source/commands/gatekeeperPartition.rst nihh20160303Sergey Koren A documentation/source/commands/generateCorrectionLayouts.rst nihh20160303Sergey Koren A documentation/source/commands/leaff.rst nihh20160303Sergey Koren A documentation/source/commands/meryl.rst nihh20160303Sergey Koren A documentation/source/commands/mhapConvert.rst nihh20160303Sergey Koren A documentation/source/commands/ovStoreBucketizer.rst nihh20160303Sergey Koren A documentation/source/commands/ovStoreBuild.rst nihh20160303Sergey Koren A documentation/source/commands/ovStoreDump.rst nihh20160303Sergey Koren A documentation/source/commands/ovStoreIndexer.rst nihh20160303Sergey Koren A documentation/source/commands/ovStoreSorter.rst nihh20160303Sergey Koren A documentation/source/commands/overlapConvert.rst nihh20160303Sergey Koren A documentation/source/commands/overlapImport.rst nihh20160303Sergey Koren A documentation/source/commands/overlapInCore.rst nihh20160303Sergey Koren A documentation/source/commands/overlapInCorePartition.rst nihh20160303Sergey Koren A documentation/source/commands/overlapPair.rst nihh20160303Sergey Koren A documentation/source/commands/prefixEditDistance-matchLimitGenerate.rst nihh20160303Sergey Koren A documentation/source/commands/readConsensus.rst nihh20160303Sergey Koren A documentation/source/commands/splitReads.rst nihh20160303Sergey Koren A documentation/source/commands/tgStoreCoverageStat.rst nihh20160303Sergey Koren A documentation/source/commands/tgStoreDump.rst nihh20160303Sergey Koren A documentation/source/commands/tgStoreFilter.rst nihh20160303Sergey Koren A documentation/source/commands/tgStoreLoad.rst nihh20160303Sergey Koren A documentation/source/commands/tgTigDisplay.rst nihh20160303Sergey Koren A documentation/source/commands/trimReads.rst nihh20160303Sergey Koren A documentation/source/commands/updateDocs.sh nihh20160303Sergey Koren A documentation/source/commands/utgcns.rst nihh20160303Sergey Koren A documentation/source/history.rst nihh20160303Sergey Koren A documentation/source/index.rst nihh20160303Sergey Koren A documentation/source/parameter-reference.rst nihh20160303Sergey Koren A documentation/source/history.rst nihh20160303Sergey Koren A documentation/source/history.rst nihh20160303Sergey Koren A documentation/source/quick-start.rst nihh20160303Sergey Koren A documentation/source/history.rst nihh20160303Sergey Koren A documentation/source/index.rst nihh20160303Sergey Koren A README.citation nihh20160303Sergey Koren A documentation/source/history.rst nihh20160303Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20160303Sergey Koren A README.md nihh20160302Sergey Koren A documentation/source/conf.py nihh20160302Sergey Koren A src/mhap/mhap-2.0.tar nihh20160302Sergey Koren A documentation/source/quick-start.rst nihh20160302Sergey Koren A src/bogart/AS_BAT_Logging.C nihh20160302Brian P. Walenz A src/stores/ovStoreBuild.C nihh20160302Sergey Koren A documentation/source/quick-start.rst nihh20160302Sergey Koren A documentation/source/tutorial.rst nihh20160302Sergey Koren A src/pipelines/canu/Configure.pm nihh20160302Sergey Koren A src/pipelines/canu/OverlapStore.pm nihh20160302Sergey Koren A src/stores/gatekeeperDumpFASTQ.C nihh20160301Brian P. Walenz A README.md nihh20160301Sergey Koren A src/falcon_sense/libfalcon/falcon.C nihh20160301Sergey Koren A src/stores/ovStoreBucketizer.C nihh20160229Sergey Koren A src/pipelines/canu.pl nihh20160229Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20160229Sergey Koren A src/pipelines/canu/Defaults.pm nihh20160229Sergey Koren A src/pipelines/canu/Gatekeeper.pm nihh20160229Sergey Koren A src/pipelines/canu/Unitig.pm nihh20160229Sergey Koren A src/pipelines/canu/Configure.pm nihh20160229Sergey Koren A src/pipelines/canu/Defaults.pm nihh20160229Sergey Koren A src/pipelines/canu/OverlapStore.pm nihh20160229Sergey Koren A src/stores/ovStoreBuild.C nihh20160229Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20160228Sergey Koren A src/falcon_sense/createFalconSenseInputs.C nihh20160225Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20160225Sergey Koren A src/falcon_sense/createFalconSenseInputs.C nihh20160225Sergey Koren A src/correction/filterCorrectionOverlaps.C nihh20160225Sergey Koren A src/correction/generateCorrectionLayouts.C nihh20160225Sergey Koren A src/AS_UTL/AS_UTL_fileIO.C nihh20160225Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C nihh20160225Brian P. Walenz A src/falcon_sense/falcon_sense.C nihh20160225Sergey Koren A src/falcon_sense/libfalcon/falcon.C nihh20160225Sergey Koren A src/falcon_sense/libfalcon/kmer_lookup.C nihh20160225Sergey Koren A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160225Brian P. Walenz A src/AS_global.C nihh20160225Brian P. Walenz A src/AS_UTL/splitToWords.H nihh20160225Brian P. Walenz A src/stores/gkStore.C nihh20160225Brian P. Walenz A src/stores/gkStore.H nihh20160225Brian P. Walenz A src/stores/ovStore.C nihh20160225Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Redo_Olaps.C nihh20160225Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C nihh20160225Brian P. Walenz A src/overlapInCore/overlapInCore.C nihh20160225Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C nihh20160225Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm.C nihh20160225Brian P. Walenz A src/meryl/meryl-dump.C nihh20160225Brian P. Walenz A src/stores/gkStore.H nihh20160225Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160225Brian P. Walenz A src/AS_UTL/intervalList.H nihh20160225Brian P. Walenz A src/pipelines/canu/OverlapMMap.pm nihh20160224Sergey Koren A src/mhap/mhap-2.0.tar nihh20160224Sergey Koren A src/mhap/mhap.mk nihh20160224Sergey Koren A src/mhap/mhapConvert.C nihh20160224Sergey Koren A src/pipelines/canu/Defaults.pm nihh20160224Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20160224Sergey Koren A src/correction/filterCorrectionOverlaps.C nihh20160224Sergey Koren A src/correction/generateCorrectionLayouts.C nihh20160224Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20160224Sergey Koren A src/pipelines/canu/Defaults.pm nihh20160224Sergey Koren A src/Makefile nihh20160224Sergey Koren A src/main.mk nihh20160224Sergey Koren A src/minimap/mmapConvert.C nihh20160224Sergey Koren A src/minimap/mmapConvert.mk nihh20160224Sergey Koren A src/pipelines/canu.pl nihh20160224Sergey Koren A src/pipelines/canu/Configure.pm nihh20160224Sergey Koren A src/pipelines/canu/Defaults.pm nihh20160224Sergey Koren A src/pipelines/canu/Meryl.pm nihh20160224Sergey Koren A src/pipelines/canu/OverlapMMap.pm nihh20160224Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20160224Sergey Koren A src/Makefile nihh20160224Sergey Koren A src/falcon_sense/falcon_sense.C nihh20160224Sergey Koren A src/falcon_sense/falcon_sense.mk nihh20160224Sergey Koren A src/falcon_sense/libfalcon/falcon.C nihh20160224Sergey Koren A src/falcon_sense/libfalcon/falcon.H nihh20160224Sergey Koren A src/falcon_sense/libfalcon/kmer_lookup.C nihh20160224Sergey Koren A src/main.mk nihh20160224Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20160224Sergey Koren A src/stores/ovStore.H nihh20160222Brian P. Walenz A src/stores/ovStoreBucketizer.C nihh20160222Brian P. Walenz A src/stores/ovStoreBuild.C nihh20160222Brian P. Walenz A src/stores/ovStoreFile.C nihh20160222Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160222Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160222Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160222Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160222Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20160222Brian P. Walenz A src/stores/tgStoreDump.C nihh20160222Sergey Koren A src/pipelines/canu/Gatekeeper.pm nihh20160222Brian P. Walenz A src/Makefile nihh20160222Brian P. Walenz A src/main.mk nihh20160222Brian P. Walenz A src/pipelines/install-perl-libraries.sh nihh20160222Brian P. Walenz A src/Makefile nihh20160222Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20160222Brian P. Walenz A src/pipelines/canu.pl nihh20160222Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20160222Brian P. Walenz A src/AS_UTL/findKeyAndValue.H nihh20160222Brian P. Walenz A src/stores/gatekeeperCreate.C nihh20160222Brian P. Walenz A src/overlapInCore/overlapPair.C nihh20160220Sergey Koren A src/stores/ovStoreBuild.C nihh20160220Sergey Koren A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160219Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20160219Brian P. Walenz A src/utgcns/utgcns.C nihh20160219Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20160219Brian P. Walenz A src/overlapInCore/overlapPair.C nihh20160219Sergey Koren A src/stores/ovStoreBuild.C nihh20160219Brian P. Walenz A src/stores/ovStoreBuild.C nihh20160219Brian P. Walenz A src/stores/ovStoreSorter.C nihh20160219Brian P. Walenz A src/stores/ovStoreBuild.C nihh20160218Brian P. Walenz A src/overlapInCore/overlapImport.C nihh20160218Brian P. Walenz A src/pipelines/canu.pl nihh20160218Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20160218Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160218Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160218Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20160218Brian P. Walenz A src/stores/ovStore.H nihh20160218Brian P. Walenz A src/stores/ovStoreBucketizer.C nihh20160218Brian P. Walenz A src/stores/ovStoreBuild.C nihh20160218Brian P. Walenz A src/stores/ovStoreFile.C nihh20160218Brian P. Walenz A src/stores/ovStoreSorter.C nihh20160218Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H nihh20160218Brian P. Walenz A src/overlapInCore/overlapPair.C nihh20160217Sergey Koren A src/overlapInCore/overlapImport.C nihh20160217Sergey Koren A src/AS_global.H nihh20160215Sergey Koren A src/Makefile nihh20160215Sergey Koren A src/overlapInCore/liboverlap/prefixEditDistance-matchLimitGenerate.C nihh20160215Sergey Koren A src/correction/generateCorrectionLayouts.C nihh20160212Sergey Koren A src/overlapInCore/overlapPair.C nihh20160212Sergey Koren A src/overlapInCore/overlapPair.mk nihh20160212Sergey Koren A src/pipelines/canu/Meryl.pm nihh20160212Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20160212Sergey Koren A src/utgcns/libNDFalcon/dw.C nihh20160212Sergey Koren A src/bogart/AS_BAT_Outputs.C nihh20160212Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160212Brian P. Walenz A src/bogart/buildGraph.C nihh20160212Brian P. Walenz A src/bogart/buildGraph.mk nihh20160212Brian P. Walenz A src/main.mk nihh20160212Brian P. Walenz A src/pipelines/canu/Output.pm nihh20160212Brian P. Walenz A src/stores/tgTig.H nihh20160212Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160211Brian P. Walenz A src/overlapBasedTrimming/trimReads.C nihh20160211Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160211Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160211Brian P. Walenz A src/stores/gatekeeperCreate.C nihh20160211Brian P. Walenz A src/stores/gatekeeperDumpMetaData.C nihh20160211Brian P. Walenz A src/stores/gkStore.C nihh20160211Brian P. Walenz A src/stores/gkStore.H nihh20160211Brian P. Walenz A src/stores/ovStore.H nihh20160211Brian P. Walenz A src/stores/ovStoreBucketizer.C nihh20160211Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20160211Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160211Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20160211Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160210Brian P. Walenz A src/Makefile nihh20160209Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20160209Brian P. Walenz A src/Makefile nihh20160209Brian P. Walenz A src/Makefile nihh20160209Brian P. Walenz A src/pipelines/canu.pl nihh20160209Brian P. Walenz A src/pipelines/canu/Grid_SGE.pm nihh20160208Brian P. Walenz A src/pipelines/canu/Grid_SGE.pm nihh20160208Brian P. Walenz A src/pipelines/canu/Grid_SGE.pm nihh20160208Brian P. Walenz A src/pipelines/canu.pl nihh20160208Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20160208Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160208Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160208Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20160208Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20160208Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160208Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20160208Brian P. Walenz A src/overlapInCore/liboverlap/Binomial_Bound.C nihh20160208Brian P. Walenz A src/main.mk nihh20160205Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps.C nihh20160205Brian P. Walenz A src/overlapErrorAdjustment/findErrors.C nihh20160205Brian P. Walenz A src/overlapInCore/liboverlap/Binomial_Bound.C nihh20160205Brian P. Walenz A src/overlapInCore/liboverlap/Binomial_Bound.H nihh20160205Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimitGenerate.C nihh20160205Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.C nihh20160205Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance.H nihh20160205Brian P. Walenz A src/stores/gkStore.H nihh20160205Brian P. Walenz A src/main.mk nihh20160204Brian P. Walenz A src/stores/gatekeeperCreate.C nihh20160204Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C nihh20160201Brian P. Walenz A src/overlapInCore/overlapPair.C nihh20160201Brian P. Walenz A src/stores/ovStore.C nihh20160201Brian P. Walenz A src/stores/ovStore.H nihh20160201Brian P. Walenz A src/stores/ovStoreStats.C nihh20160201Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C nihh20160201Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H nihh20160201Brian P. Walenz A src/meryl/meryl-build.C nihh20160129Brian P. Walenz A src/meryl/meryl-estimate.C nihh20160129Brian P. Walenz A src/meryl/meryl.H nihh20160129Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160129Brian P. Walenz A src/Makefile nihh20160129Brian P. Walenz A src/stores/ovStore.H nihh20160129Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160128Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160128Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160128Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160128Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160128Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.H nihh20160128Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C nihh20160128Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H nihh20160128Brian P. Walenz A src/bogart/bogart.C nihh20160128Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20160128Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20160128Sergey Koren A src/pipelines/canu.pl nihh20160127Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20160127Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160127Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20160127Brian P. Walenz A src/stores/tgStoreFilter.C nihh20160127Brian P. Walenz A src/bogart/bogart.C nihh20160127Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160127Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160127Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160127Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20160127Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20160127Brian P. Walenz A src/stores/tgTigSizeAnalysis.C nihh20160127Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160127Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.H nihh20160127Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C nihh20160127Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H nihh20160127Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160127Brian P. Walenz A src/bogart/AS_BAT_PromoteToSingleton.C nihh20160127Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160127Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C nihh20160127Brian P. Walenz A src/bogart/bogart.C nihh20160127Brian P. Walenz A src/correction/generateCorrectionLayouts.C nihh20160127Brian P. Walenz A src/pipelines/canu/Output.pm nihh20160127Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20160127Brian P. Walenz A src/stores/gkStore.H nihh20160127Brian P. Walenz A src/stores/tgStore.H nihh20160127Brian P. Walenz A src/stores/tgStoreDump.C nihh20160127Brian P. Walenz A src/stores/tgStoreFilter.C nihh20160127Brian P. Walenz A src/stores/tgTig.C nihh20160127Brian P. Walenz A src/stores/tgTig.H nihh20160127Brian P. Walenz A src/stores/tgTigSizeAnalysis.C nihh20160127Brian P. Walenz A src/stores/tgTigSizeAnalysis.H nihh20160127Brian P. Walenz A src/pipelines/canu/Grid_SGE.pm nihh20160122Brian P. Walenz A src/canu_version_update.pl nihh20160122Brian P. Walenz A src/pipelines/canu/Output.pm nihh20160122Brian P. Walenz A documentation/source/parameter-reference.rst nihh20160120Brian P. Walenz A src/pipelines/canu/Grid_SGE.pm nihh20160120Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160120Brian P. Walenz A src/pipelines/canu.pl nihh20160120Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20160120Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160120Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160120Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20160120Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20160120Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20160120Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20160120Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20160120Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20160120Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20160120Brian P. Walenz A src/overlapBasedTrimming/splitReads-trimBad.C nihh20160119Brian P. Walenz A src/overlapBasedTrimming/splitReads.C nihh20160119Brian P. Walenz A src/pipelines/canu/HTML.pm nihh20160119Brian P. Walenz A src/overlapBasedTrimming/trimReads.C nihh20160119Brian P. Walenz A src/overlapBasedTrimming/trimReads.C nihh20160119Brian P. Walenz A src/overlapBasedTrimming/trimStat.H nihh20160119Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20160119Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160119Brian P. Walenz A src/pipelines/canu.pl nihh20160117Brian P. Walenz A src/overlapBasedTrimming/trimReads.C nihh20160117Brian P. Walenz A src/pipelines/canu/HTML.pm nihh20160117Brian P. Walenz A src/pipelines/canu/HTML.pm nihh20160117Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm nihh20160117Brian P. Walenz A src/pipelines/canu.pl nihh20160117Brian P. Walenz A src/pipelines/canu.pl nihh20160115Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160115Brian P. Walenz A src/Makefile nihh20160115Brian P. Walenz A src/utgcns/libcns/abBead.H nihh20160114Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160114Brian P. Walenz A src/pipelines/canu/HTML.pm nihh20160114Brian P. Walenz A src/pipelines/canu.pl nihh20160114Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160114Brian P. Walenz A src/pipelines/canu/Grid.pm nihh20160114Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20160114Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160114Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20160114Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160112Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160112Brian P. Walenz A src/pipelines/canu.pl nihh20160112Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160112Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20160112Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160112Brian P. Walenz A src/pipelines/canu.pl nihh20160112Brian P. Walenz A src/AS_global.C nihh20160112Brian P. Walenz A src/correction/filterCorrectionOverlaps.C nihh20160112Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160112Brian P. Walenz A src/pipelines/canu/HTML.pm nihh20160112Brian P. Walenz A src/correction/filterCorrectionOverlaps.C nihh20160112Brian P. Walenz A src/correction/generateCorrectionLayouts.C nihh20160112Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160111Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20160111Brian P. Walenz A src/AS_RUN/fragmentDepth.C nihh20160111Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.C nihh20160111Brian P. Walenz A src/AS_UTL/AS_UTL_alloc.H nihh20160111Brian P. Walenz A src/AS_UTL/AS_UTL_decodeRange.C nihh20160111Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.C nihh20160111Brian P. Walenz A src/AS_UTL/AS_UTL_fileIO.H nihh20160111Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.C nihh20160111Brian P. Walenz A src/AS_UTL/AS_UTL_reverseComplement.H nihh20160111Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.C nihh20160111Brian P. Walenz A src/AS_UTL/AS_UTL_stackTrace.H nihh20160111Brian P. Walenz A src/AS_UTL/memoryMappedFile.H nihh20160111Brian P. Walenz A src/AS_UTL/stddev.C nihh20160111Brian P. Walenz A src/AS_UTL/testRand.C nihh20160111Brian P. Walenz A src/AS_global.C nihh20160111Brian P. Walenz A src/AS_global.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Breaking.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Breaking.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_IntersectSplit.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Joining.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Joining.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Logging.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Logging.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Outputs.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PlaceZombies.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PromoteToSingleton.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Unitig.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddAndPlaceFrag.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddFrag.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C nihh20160111Brian P. Walenz A src/bogart/addReadsToUnitigs.C nihh20160111Brian P. Walenz A src/bogart/analyzeBest.C nihh20160111Brian P. Walenz A src/bogart/bogart.C nihh20160111Brian P. Walenz A src/bogus/bogus.C nihh20160111Brian P. Walenz A src/bogus/bogusUtil.C nihh20160111Brian P. Walenz A src/bogus/bogusUtil.H nihh20160111Brian P. Walenz A src/bogus/bogusness.C nihh20160111Brian P. Walenz A src/correction/generateCorrectionLayouts.C nihh20160111Brian P. Walenz A src/correction/readConsensus.C nihh20160111Brian P. Walenz A src/erateEstimate/erateEstimate.C nihh20160111Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.C nihh20160111Brian P. Walenz A src/falcon_sense/outputFalcon.C nihh20160111Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C nihh20160111Brian P. Walenz A src/fastq-utilities/fastqSample.C nihh20160111Brian P. Walenz A src/fastq-utilities/fastqSimulate-sort.C nihh20160111Brian P. Walenz A src/fastq-utilities/fastqSimulate.C nihh20160111Brian P. Walenz A src/merTrim/merTrim.C nihh20160111Brian P. Walenz A src/merTrim/merTrimAdapter.C nihh20160111Brian P. Walenz A src/merTrim/merTrimApply.C nihh20160111Brian P. Walenz A src/merTrim/merTrimResult.H nihh20160111Brian P. Walenz A src/mercy/mercy-regions.C nihh20160111Brian P. Walenz A src/mercy/mercy.C nihh20160111Brian P. Walenz A src/meryl/estimate-mer-threshold.C nihh20160111Brian P. Walenz A src/overlapBasedTrimming/adjustFlipped.C nihh20160111Brian P. Walenz A src/overlapBasedTrimming/adjustNormal.C nihh20160111Brian P. Walenz A src/overlapBasedTrimming/splitReads-subReads.C nihh20160111Brian P. Walenz A src/overlapBasedTrimming/splitReads.C nihh20160111Brian P. Walenz A src/overlapBasedTrimming/trimReads-bestEdge.C nihh20160111Brian P. Walenz A src/overlapBasedTrimming/trimReads-largestCovered.C nihh20160111Brian P. Walenz A src/overlapBasedTrimming/trimReads-quality.C nihh20160111Brian P. Walenz A src/overlapBasedTrimming/trimReads.C nihh20160111Brian P. Walenz A src/overlapBasedTrimming/trimReads.H nihh20160111Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps.C nihh20160111Brian P. Walenz A src/overlapErrorAdjustment/findErrors.C nihh20160111Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-extend.C nihh20160111Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C nihh20160111Brian P. Walenz A src/overlapInCore/overlapInCore-Find_Overlaps.C nihh20160111Brian P. Walenz A src/overlapInCore/overlapInCore-Output.C nihh20160111Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C nihh20160111Brian P. Walenz A src/overlapInCore/overlapInCore-Process_String_Overlaps.C nihh20160111Brian P. Walenz A src/overlapInCore/overlapInCore.C nihh20160111Brian P. Walenz A src/overlapInCore/overlapInCore.H nihh20160111Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C nihh20160111Brian P. Walenz A src/overlapInCore/overlapPair.C nihh20160111Brian P. Walenz A src/stores/gatekeeperCreate.C nihh20160111Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C nihh20160111Brian P. Walenz A src/stores/gatekeeperDumpMetaData.C nihh20160111Brian P. Walenz A src/stores/gatekeeperPartition.C nihh20160111Brian P. Walenz A src/stores/ovStore.C nihh20160111Brian P. Walenz A src/stores/ovStore.H nihh20160111Brian P. Walenz A src/stores/ovStoreBucketizer.C nihh20160111Brian P. Walenz A src/stores/ovStoreBuild.C nihh20160111Brian P. Walenz A src/stores/ovStoreDump.C nihh20160111Brian P. Walenz A src/stores/ovStoreFile.C nihh20160111Brian P. Walenz A src/stores/ovStoreIndexer.C nihh20160111Brian P. Walenz A src/stores/ovStoreSorter.C nihh20160111Brian P. Walenz A src/stores/ovStoreStats.C nihh20160111Brian P. Walenz A src/stores/tgStore.C nihh20160111Brian P. Walenz A src/stores/tgStore.H nihh20160111Brian P. Walenz A src/stores/tgStoreCoverageStat.C nihh20160111Brian P. Walenz A src/stores/tgStoreDump.C nihh20160111Brian P. Walenz A src/stores/tgStoreFilter.C nihh20160111Brian P. Walenz A src/stores/tgStoreLoad.C nihh20160111Brian P. Walenz A src/stores/tgTig.H nihh20160111Brian P. Walenz A src/stores/tgTigDisplay.C nihh20160111Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C nihh20160111Brian P. Walenz A src/stores/tgTigSizeAnalysis.C nihh20160111Brian P. Walenz A src/stores/tgTigSizeAnalysis.H nihh20160111Brian P. Walenz A src/utgcns/libNDalign/NDalgorithm-extend.C nihh20160111Brian P. Walenz A src/utgcns/libNNalign/NNalgorithm.C nihh20160111Brian P. Walenz A src/utgcns/libNNalign/NNalign.C nihh20160111Brian P. Walenz A src/utgcns/libNNalign/NNalign.H nihh20160111Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20160111Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C nihh20160111Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C nihh20160111Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C nihh20160111Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C nihh20160111Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20160111Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20160111Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H nihh20160111Brian P. Walenz A src/utgcns/stashContains.C nihh20160111Brian P. Walenz A src/utgcns/utgcns.C nihh20160111Brian P. Walenz A src/AS_global.C nihh20160111Brian P. Walenz A src/Makefile nihh20160111Brian P. Walenz A src/canu_version_update.pl nihh20160111Brian P. Walenz A src/pipelines/canu.pl nihh20160111Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160111Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160111Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160111Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Logging.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_Logging.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_MergeSplitJoin.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C nihh20160111Brian P. Walenz A src/bogart/AS_BAT_RepeatJunctionEvidence.H nihh20160111Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C nihh20160111Brian P. Walenz A src/bogart/bogart.C nihh20160111Brian P. Walenz A src/bogart/bogart.mk nihh20160111Brian P. Walenz A src/stores/tgStoreFilter.C nihh20160108Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20160107Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160107Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160107Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20160107Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20160107Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C nihh20160107Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160107Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20160107Brian P. Walenz A src/pipelines/canu.pl nihh20160106Sergey Koren A src/utgcns/stashContains.C nihh20160106Brian P. Walenz A src/utgcns/stashContains.H nihh20160106Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20160105Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20160105Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20160105Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C nihh20160105Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C nihh20160105Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20160105Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20160105Brian P. Walenz A src/utgcns/libNDalign/NDalign.C nihh20160105Brian P. Walenz A src/stores/tgTig.H nihh20160105Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160105Brian P. Walenz A documentation/source/quick-start.rst nihh20160105Sergey Koren A src/pipelines/canu/Consensus.pm nihh20160105Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160105Brian P. Walenz A src/utgcns/utgcns.C nihh20160105Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160104Brian P. Walenz A src/AS_global.H nihh20160104Brian P. Walenz A src/stores/tgStoreFilter.C nihh20160104Brian P. Walenz A src/utgcns/libNDFalcon/dw.H nihh20160104Brian P. Walenz A src/utgcns/libpbutgcns/Alignment.H nihh20160104Brian P. Walenz A src/utgcns/libpbutgcns/Alignment.H nihh20160104Sergey Koren A documentation/source/quick-start.rst nihh20151230Sergey Koren A documentation/source/tutorial.rst nihh20151230Sergey Koren A documentation/source/tutorial.rst nihh20151230Sergey Koren A documentation/source/quick-start.rst nihh20151229Sergey Koren A src/utgcns/libpbutgcns/LICENSE nihh20151229Sergey Koren A documentation/source/quick-start.rst nihh20151228Sergey Koren A documentation/source/quick-start.rst nihh20151228Sergey Koren A documentation/source/quick-start.rst nihh20151228Sergey Koren A src/pipelines/canu/Consensus.pm nihh20151228Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151228Sergey Koren A src/utgcns/libcns/unitigConsensus.C nihh20151228Sergey Koren A src/utgcns/libpbutgcns/SimpleAligner.C nihh20151228Sergey Koren A src/utgcns/libpbutgcns/SimpleAligner.H nihh20151228Sergey Koren A src/main.mk nihh20151228Sergey Koren A src/pipelines/canu.pl nihh20151228Sergey Koren A src/pipelines/canu/Configure.pm nihh20151228Sergey Koren A src/pipelines/canu/Consensus.pm nihh20151228Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151228Sergey Koren A src/utgcns/libNDFalcon/LICENSE nihh20151228Sergey Koren A src/utgcns/libNDFalcon/dw.C nihh20151228Sergey Koren A src/utgcns/libNDFalcon/dw.H nihh20151228Sergey Koren A src/utgcns/libcns/unitigConsensus.C nihh20151228Sergey Koren A src/utgcns/libcns/unitigConsensus.H nihh20151228Sergey Koren A src/utgcns/libpbutgcns/.gitignore nihh20151228Sergey Koren A src/utgcns/libpbutgcns/Alignment.C nihh20151228Sergey Koren A src/utgcns/libpbutgcns/Alignment.H nihh20151228Sergey Koren A src/utgcns/libpbutgcns/AlnGraphBoost.C nihh20151228Sergey Koren A src/utgcns/libpbutgcns/AlnGraphBoost.H nihh20151228Sergey Koren A src/utgcns/libpbutgcns/LICENSE nihh20151228Sergey Koren A src/utgcns/libpbutgcns/README.md nihh20151228Sergey Koren A src/utgcns/libpbutgcns/SimpleAligner.C nihh20151228Sergey Koren A src/utgcns/libpbutgcns/SimpleAligner.H nihh20151228Sergey Koren A src/utgcns/utgcns.C nihh20151228Sergey Koren A src/utgcns/utgcns.mk nihh20151228Sergey Koren A src/utgcns/libcns/abAbacus-addRead.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151223Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abSequence.H nihh20151223Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151223Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abBead.H nihh20151223Brian P. Walenz A src/utgcns/libcns/abColumn.H nihh20151223Brian P. Walenz A src/utgcns/utgcns.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151223Brian P. Walenz A src/utgcns/libcns/abSequence.H nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151223Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151223Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H nihh20151223Brian P. Walenz A src/utgcns/utgcns.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C nihh20151223Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151222Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C nihh20151222Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151222Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C nihh20151222Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C nihh20151222Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C nihh20151222Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151222Brian P. Walenz A src/utgcns/libcns/abBead.H nihh20151222Brian P. Walenz A src/utgcns/libcns/abColumn.H nihh20151222Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C nihh20151222Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C nihh20151222Brian P. Walenz A src/pipelines/canu.pl nihh20151222Sergey Koren A src/pipelines/canu/Consensus.pm nihh20151222Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151222Sergey Koren A src/utgcns/libcns/unitigConsensus.C nihh20151222Sergey Koren A src/utgcns/libcns/abAbacus-addRead.C nihh20151221Brian P. Walenz A src/utgcns/libcns/abAbacus-appendBases.C nihh20151221Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151221Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C nihh20151221Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C nihh20151221Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C nihh20151221Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C nihh20151221Brian P. Walenz A src/utgcns/libcns/abAbacus.C nihh20151221Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151221Brian P. Walenz A src/utgcns/libcns/abBead.H nihh20151221Brian P. Walenz A src/utgcns/libcns/abColumn.H nihh20151221Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C nihh20151221Brian P. Walenz A src/utgcns/libcns/abSequence.H nihh20151221Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151221Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H nihh20151221Brian P. Walenz A src/utgcns/utgcns.C nihh20151221Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151221Brian P. Walenz A src/bogart/bogart.mk nihh20151221Brian P. Walenz A src/bogus/bogus.mk nihh20151221Brian P. Walenz A src/bogus/bogusness.mk nihh20151221Brian P. Walenz A src/correction/filterCorrectionOverlaps.mk nihh20151221Brian P. Walenz A src/correction/generateCorrectionLayouts.mk nihh20151221Brian P. Walenz A src/correction/readConsensus.mk nihh20151221Brian P. Walenz A src/erateEstimate/erateEstimate.mk nihh20151221Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.mk nihh20151221Brian P. Walenz A src/fastq-utilities/fastqAnalyze.mk nihh20151221Brian P. Walenz A src/fastq-utilities/fastqSample.mk nihh20151221Brian P. Walenz A src/fastq-utilities/fastqSimulate-sort.mk nihh20151221Brian P. Walenz A src/fastq-utilities/fastqSimulate.mk nihh20151221Brian P. Walenz A src/main.mk nihh20151221Brian P. Walenz A src/meryl/estimate-mer-threshold.mk nihh20151221Brian P. Walenz A src/meryl/leaff.mk nihh20151221Brian P. Walenz A src/meryl/meryl.mk nihh20151221Brian P. Walenz A src/meryl/simple.mk nihh20151221Brian P. Walenz A src/mhap/mhapConvert.mk nihh20151221Brian P. Walenz A src/overlapBasedTrimming/splitReads.mk nihh20151221Brian P. Walenz A src/overlapBasedTrimming/trimReads.mk nihh20151221Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps.mk nihh20151221Brian P. Walenz A src/overlapErrorAdjustment/findErrors.mk nihh20151221Brian P. Walenz A src/overlapInCore/liboverlap/prefixEditDistance-matchLimitGenerate.mk nihh20151221Brian P. Walenz A src/overlapInCore/overlapConvert.mk nihh20151221Brian P. Walenz A src/overlapInCore/overlapImport.mk nihh20151221Brian P. Walenz A src/overlapInCore/overlapInCore.mk nihh20151221Brian P. Walenz A src/overlapInCore/overlapInCorePartition.mk nihh20151221Brian P. Walenz A src/overlapInCore/overlapPair.mk nihh20151221Brian P. Walenz A src/stores/gatekeeperCreate.mk nihh20151221Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.mk nihh20151221Brian P. Walenz A src/stores/gatekeeperDumpMetaData.mk nihh20151221Brian P. Walenz A src/stores/gatekeeperPartition.mk nihh20151221Brian P. Walenz A src/stores/ovStoreBucketizer.mk nihh20151221Brian P. Walenz A src/stores/ovStoreBuild.mk nihh20151221Brian P. Walenz A src/stores/ovStoreDump.mk nihh20151221Brian P. Walenz A src/stores/ovStoreIndexer.mk nihh20151221Brian P. Walenz A src/stores/ovStoreSorter.mk nihh20151221Brian P. Walenz A src/stores/ovStoreStats.mk nihh20151221Brian P. Walenz A src/stores/tgStoreCoverageStat.mk nihh20151221Brian P. Walenz A src/stores/tgStoreDump.mk nihh20151221Brian P. Walenz A src/stores/tgStoreFilter.mk nihh20151221Brian P. Walenz A src/stores/tgStoreLoad.mk nihh20151221Brian P. Walenz A src/stores/tgTigDisplay.mk nihh20151221Brian P. Walenz A src/utgcns/utgcns.mk nihh20151221Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm nihh20151221Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C nihh20151218Brian P. Walenz A src/utgcns/libcns/abAbacus-appendBases.C nihh20151218Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151218Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C nihh20151218Brian P. Walenz A src/utgcns/libcns/abAbacus-mergeRefine.C nihh20151218Brian P. Walenz A src/utgcns/libcns/abAbacus-refine.C nihh20151218Brian P. Walenz A src/utgcns/libcns/abAbacus-refreshMultiAlign.C nihh20151218Brian P. Walenz A src/utgcns/libcns/abAbacus.C nihh20151218Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151218Brian P. Walenz A src/utgcns/libcns/abBead.H nihh20151218Brian P. Walenz A src/utgcns/libcns/abColumn.C nihh20151218Brian P. Walenz A src/utgcns/libcns/abColumn.H nihh20151218Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C nihh20151218Brian P. Walenz A src/utgcns/libcns/abSequence.H nihh20151218Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151218Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H nihh20151218Brian P. Walenz A src/utgcns/utgcns.C nihh20151218Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151217Sergey Koren A src/pipelines/canu/OverlapStore.pm nihh20151217Sergey Koren A documentation/source/quick-start.rst nihh20151217Sergey Koren A src/utgcns/libcns/unitigConsensus.C nihh20151217Sergey Koren A src/pipelines/canu/Configure.pm nihh20151217Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151216Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151216Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151216Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20151216Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20151216Sergey Koren A src/pipelines/canu/Unitig.pm nihh20151216Sergey Koren A src/pipelines/canu/Consensus.pm nihh20151216Sergey Koren A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151215Brian P. Walenz A src/pipelines/canu.pl nihh20151215Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151215Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20151215Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151215Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151215Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20151215Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151215Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20151215Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20151215Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151215Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20151215Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151215Sergey Koren A src/AS_UTL/memoryMappedFile.H nihh20151215Sergey Koren A src/stores/ovStore.C nihh20151215Sergey Koren A src/pipelines/canu/Execution.pm nihh20151214Sergey Koren A src/pipelines/canu/Grid_PBSTorque.pm nihh20151214Sergey Koren A src/pipelines/canu/Grid_Slurm.pm nihh20151214Sergey Koren A documentation/source/quick-start.rst nihh20151211Sergey Koren A src/utgcns/libcns/abAbacus-addRead.C nihh20151211Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151209Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20151209Sergey Koren A src/pipelines/canu/Configure.pm nihh20151209Brian P. Walenz A documentation/source/command-reference.rst nihh20151209Sergey Koren A documentation/source/commands/bogart.rst nihh20151209Sergey Koren A documentation/source/commands/bogus.rst nihh20151209Sergey Koren A documentation/source/commands/canu.rst nihh20151209Sergey Koren A documentation/source/commands/correctOverlaps.rst nihh20151209Sergey Koren A documentation/source/commands/createFalconSenseInputs.rst nihh20151209Sergey Koren A documentation/source/commands/erateEstimate.rst nihh20151209Sergey Koren A documentation/source/commands/estimate-mer-threshold.rst nihh20151209Sergey Koren A documentation/source/commands/fastqAnalyze.rst nihh20151209Sergey Koren A documentation/source/commands/fastqSample.rst nihh20151209Sergey Koren A documentation/source/commands/fastqSimulate-sort.rst nihh20151209Sergey Koren A documentation/source/commands/fastqSimulate.rst nihh20151209Sergey Koren A documentation/source/commands/filterCorrectionOverlaps.rst nihh20151209Sergey Koren A documentation/source/commands/findErrors.rst nihh20151209Sergey Koren A documentation/source/commands/gatekeeperCreate.rst nihh20151209Sergey Koren A documentation/source/commands/gatekeeperDumpFASTQ.rst nihh20151209Sergey Koren A documentation/source/commands/gatekeeperDumpMetaData.rst nihh20151209Sergey Koren A documentation/source/commands/gatekeeperPartition.rst nihh20151209Sergey Koren A documentation/source/commands/generateCorrectionLayouts.rst nihh20151209Sergey Koren A documentation/source/commands/leaff.rst nihh20151209Sergey Koren A documentation/source/commands/meryl.rst nihh20151209Sergey Koren A documentation/source/commands/mhapConvert.rst nihh20151209Sergey Koren A documentation/source/commands/ovStoreBucketizer.rst nihh20151209Sergey Koren A documentation/source/commands/ovStoreBuild.rst nihh20151209Sergey Koren A documentation/source/commands/ovStoreDump.rst nihh20151209Sergey Koren A documentation/source/commands/ovStoreIndexer.rst nihh20151209Sergey Koren A documentation/source/commands/ovStoreSorter.rst nihh20151209Sergey Koren A documentation/source/commands/overlapConvert.rst nihh20151209Sergey Koren A documentation/source/commands/overlapImport.rst nihh20151209Sergey Koren A documentation/source/commands/overlapInCore.rst nihh20151209Sergey Koren A documentation/source/commands/overlapInCorePartition.rst nihh20151209Sergey Koren A documentation/source/commands/overlapPair.rst nihh20151209Sergey Koren A documentation/source/commands/prefixEditDistance-matchLimitGenerate.rst nihh20151209Sergey Koren A documentation/source/commands/readConsensus.rst nihh20151209Sergey Koren A documentation/source/commands/simple.rst nihh20151209Sergey Koren A documentation/source/commands/splitReads.rst nihh20151209Sergey Koren A documentation/source/commands/tgStoreCoverageStat.rst nihh20151209Sergey Koren A documentation/source/commands/tgStoreDump.rst nihh20151209Sergey Koren A documentation/source/commands/tgStoreFilter.rst nihh20151209Sergey Koren A documentation/source/commands/tgStoreLoad.rst nihh20151209Sergey Koren A documentation/source/commands/tgTigDisplay.rst nihh20151209Sergey Koren A documentation/source/commands/trimReads.rst nihh20151209Sergey Koren A documentation/source/commands/utgcns.rst nihh20151209Sergey Koren A documentation/source/conf.py nihh20151209Sergey Koren A documentation/source/index.rst nihh20151209Sergey Koren A documentation/source/parameter-reference.rst nihh20151209Sergey Koren A documentation/source/quick-start.rst nihh20151209Sergey Koren A documentation/source/tutorial.rst nihh20151209Sergey Koren A src/stores/gkStore.C nihh20151209Sergey Koren A src/stores/gatekeeperDumpFASTQ.C nihh20151208Brian P. Walenz A src/pipelines/canu/Gatekeeper.pm nihh20151208Brian P. Walenz A src/stores/gkStore.C nihh20151208Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151208Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151208Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20151208Brian P. Walenz A src/falcon_sense/falcon_sense.py nihh20151208Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20151207Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20151207Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20151207Brian P. Walenz A src/pipelines/canu.pl nihh20151207Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151207Brian P. Walenz A src/pipelines/canu/Grid.pm nihh20151207Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151207Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151207Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20151207Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151207Brian P. Walenz A src/pipelines/canu/Grid_LSF.pm nihh20151207Brian P. Walenz A src/pipelines/canu/Grid_PBSTorque.pm nihh20151207Brian P. Walenz A src/pipelines/canu/Grid_SGE.pm nihh20151207Brian P. Walenz A src/pipelines/canu/Grid_Slurm.pm nihh20151207Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151207Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20151207Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20151207Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151207Brian P. Walenz A src/bogart/addReadsToUnitigs.C nihh20151207Brian P. Walenz A src/bogart/analyzeBest.C nihh20151207Brian P. Walenz A src/bogart/bogart.C nihh20151207Brian P. Walenz A src/correction/filterCorrectionOverlaps.C nihh20151207Brian P. Walenz A src/correction/generateCorrectionLayouts.C nihh20151207Brian P. Walenz A src/correction/readConsensus.C nihh20151207Brian P. Walenz A src/erateEstimate/erateEstimate.C nihh20151207Brian P. Walenz A src/falcon_sense/createFalconSenseInputs.C nihh20151207Brian P. Walenz A src/merTrim/merTrim.C nihh20151207Brian P. Walenz A src/merTrim/merTrimApply.C nihh20151207Brian P. Walenz A src/meryl/leaff-simulate.C nihh20151207Brian P. Walenz A src/meryl/leaff.C nihh20151207Brian P. Walenz A src/meryl/libleaff/gkStoreFile.C nihh20151207Brian P. Walenz A src/overlapBasedTrimming/splitReads.C nihh20151207Brian P. Walenz A src/overlapBasedTrimming/trimReads.C nihh20151207Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps.C nihh20151207Brian P. Walenz A src/overlapErrorAdjustment/findErrors.C nihh20151207Brian P. Walenz A src/overlapInCore/overlapConvert.C nihh20151207Brian P. Walenz A src/overlapInCore/overlapImport.C nihh20151207Brian P. Walenz A src/overlapInCore/overlapInCore.C nihh20151207Brian P. Walenz A src/overlapInCore/overlapInCorePartition.C nihh20151207Brian P. Walenz A src/overlapInCore/overlapPair.C nihh20151207Brian P. Walenz A src/stores/gatekeeperCreate.C nihh20151207Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C nihh20151207Brian P. Walenz A src/stores/gatekeeperDumpMetaData.C nihh20151207Brian P. Walenz A src/stores/gatekeeperPartition.C nihh20151207Brian P. Walenz A src/stores/gkStore.C nihh20151207Brian P. Walenz A src/stores/gkStore.H nihh20151207Brian P. Walenz A src/stores/ovStoreBucketizer.C nihh20151207Brian P. Walenz A src/stores/ovStoreBuild.C nihh20151207Brian P. Walenz A src/stores/ovStoreDump.C nihh20151207Brian P. Walenz A src/stores/ovStoreStats.C nihh20151207Brian P. Walenz A src/stores/tgStoreCoverageStat.C nihh20151207Brian P. Walenz A src/stores/tgStoreDump.C nihh20151207Brian P. Walenz A src/stores/tgStoreFilter.C nihh20151207Brian P. Walenz A src/stores/tgStoreLoad.C nihh20151207Brian P. Walenz A src/stores/tgTigDisplay.C nihh20151207Brian P. Walenz A src/utgcns/utgcns.C nihh20151207Brian P. Walenz A src/stores/gatekeeperCreate.C nihh20151207Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C nihh20151207Brian P. Walenz A src/stores/gkStore.C nihh20151207Brian P. Walenz A src/stores/gkStore.H nihh20151207Brian P. Walenz A src/stores/gkStoreEncode.C nihh20151207Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C nihh20151207Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151204Sergey Koren A src/pipelines/canu/Meryl.pm nihh20151204Sergey Koren A src/pipelines/canu/Execution.pm nihh20151204Sergey Koren A src/stores/gkStore.C nihh20151204Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20151204Sergey Koren A src/pipelines/canu/Execution.pm nihh20151204Sergey Koren A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151203Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151203Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151203Brian P. Walenz A src/stores/tgTigMultiAlignDisplay.C nihh20151203Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C nihh20151203Brian P. Walenz A src/main.mk nihh20151203Brian P. Walenz A src/stores/gatekeeperCreate.C nihh20151203Brian P. Walenz A src/stores/gkStore.C nihh20151203Brian P. Walenz A src/stores/gkStore.H nihh20151203Brian P. Walenz A src/stores/gkStoreEncode.C nihh20151203Brian P. Walenz A src/fastq-utilities/fastqAnalyze.C nihh20151203Brian P. Walenz A src/pipelines/canu/Output.pm nihh20151202Sergey Koren A documentation/source/quick-start.rst nihh20151202Sergey Koren A documentation/source/quick-start.rst nihh20151202Sergey Koren A documentation/source/quick-start.rst nihh20151202Sergey Koren A documentation/source/quick-start.rst nihh20151202Sergey Koren A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151202Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20151202Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20151202Sergey Koren A src/pipelines/canu.pl nihh20151201Sergey Koren A src/pipelines/canu.pl nihh20151201Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20151201Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20151201Brian P. Walenz A src/pipelines/canu.pl nihh20151201Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151201Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20151201Brian P. Walenz A src/pipelines/canu.pl nihh20151201Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151201Brian P. Walenz A src/stores/tgTig.H nihh20151201Brian P. Walenz A src/utgcns/libcns/abMultiAlign.C nihh20151201Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151201Brian P. Walenz A src/utgcns/stashContains.C nihh20151201Brian P. Walenz A src/utgcns/utgcns.C nihh20151201Brian P. Walenz A src/stores/tgTig.C nihh20151130Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20151130Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151130Brian P. Walenz A src/pipelines/canu/Grid_LSF.pm nihh20151130Sergey Koren A src/pipelines/canu/Grid_PBSTorque.pm nihh20151130Sergey Koren A src/pipelines/canu/Grid_SGE.pm nihh20151130Sergey Koren A src/pipelines/canu/Grid_Slurm.pm nihh20151130Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151130Sergey Koren A src/pipelines/canu/Execution.pm nihh20151130Sergey Koren A src/pipelines/canu/Grid_LSF.pm nihh20151130Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151130Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20151130Sergey Koren A src/pipelines/canu/Meryl.pm nihh20151130Sergey Koren A src/pipelines/canu/Grid_Slurm.pm nihh20151130Sergey Koren A src/pipelines/canu.pl nihh20151127Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Grid_SGE.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Grid_LSF.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Grid_PBSTorque.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Grid_Slurm.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151127Sergey Koren A src/pipelines/canu/Meryl.pm nihh20151127Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20151127Sergey Koren A src/Makefile nihh20151127Brian P. Walenz A src/pipelines/canu.pl nihh20151127Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Grid.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Grid_LSF.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Grid_PBSTorque.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Grid_SGE.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Grid_Slurm.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151127Brian P. Walenz A src/pipelines/canu.pl nihh20151127Sergey Koren A src/correction/generateCorrectionLayouts.C nihh20151127Brian P. Walenz A src/pipelines/canu.pl nihh20151127Sergey Koren A src/pipelines/canu/Meryl.pm nihh20151127Sergey Koren A src/pipelines/canu/Unitig.pm nihh20151127Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151127Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20151127Brian P. Walenz A src/stores/tgTig.C nihh20151125Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151125Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20151125Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151125Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20151125Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151125Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151125Sergey Koren A README.md nihh20151125Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151125Sergey Koren A src/pipelines/canu/Execution.pm nihh20151125Sergey Koren A src/utgcns/libcns/abAbacus-appendBases.C nihh20151125Brian P. Walenz A src/pipelines/canu/CorrectReads.pm nihh20151124Sergey Koren A src/utgcns/utgcns.C nihh20151124Brian P. Walenz A src/stores/tgStoreDump.C nihh20151124Brian P. Walenz A src/stores/gatekeeperDumpMetaData.C nihh20151124Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151124Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151124Brian P. Walenz A src/stores/gatekeeperCreate.C nihh20151123Brian P. Walenz A src/stores/gkStore.C nihh20151123Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.C nihh20151123Brian P. Walenz A src/AS_UTL/AS_UTL_fasta.H nihh20151123Brian P. Walenz A src/merTrim/merTrim.C nihh20151123Brian P. Walenz A src/overlapBasedTrimming/trimReads-quality.C nihh20151123Brian P. Walenz A src/overlapInCore/overlapInCore-Build_Hash_Index.C nihh20151123Brian P. Walenz A src/overlapInCore/overlapInCore-Process_Overlaps.C nihh20151123Brian P. Walenz A src/overlapInCore/overlapInCore.H nihh20151123Brian P. Walenz A src/stores/gatekeeperCreate.C nihh20151123Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C nihh20151123Brian P. Walenz A src/stores/gkStore.C nihh20151123Brian P. Walenz A src/stores/gkStore.H nihh20151123Brian P. Walenz A src/stores/tgStoreDump.C nihh20151123Brian P. Walenz A src/stores/tgTig.C nihh20151123Brian P. Walenz A src/stores/tgTig.H nihh20151123Brian P. Walenz A src/utgcns/libcns/abAbacus-addRead.C nihh20151123Brian P. Walenz A src/utgcns/libcns/abAbacus-baseCall.C nihh20151123Brian P. Walenz A src/utgcns/libcns/abAbacus.C nihh20151123Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151123Brian P. Walenz A src/utgcns/utgcns.C nihh20151123Brian P. Walenz A src/main.mk nihh20151123Brian P. Walenz A src/utgcns/libcns/abAbacus-appendBases.C nihh20151123Brian P. Walenz A src/utgcns/libcns/abAbacus.H nihh20151123Brian P. Walenz A src/utgcns/libcns/abIDs.H nihh20151123Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151123Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151123Brian P. Walenz A src/utgcns/libcns/abSequence.H nihh20151123Brian P. Walenz A src/stores/tgTig.H nihh20151123Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20151121Sergey Koren A src/pipelines/canu.pl nihh20151121Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151121Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20151120Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151120Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20151120Sergey Koren A src/pipelines/canu.pl nihh20151120Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20151120Brian P. Walenz A src/overlapInCore/overlapInCore.C nihh20151120Sergey Koren A src/overlapInCore/overlapInCore.H nihh20151120Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151120Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151120Brian P. Walenz A src/pipelines/canu/Consensus.pm nihh20151120Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20151120Brian P. Walenz A src/utgcns/libcns/abAbacus-applyAlignment.C nihh20151120Brian P. Walenz A src/utgcns/libcns/unitigConsensus.C nihh20151120Brian P. Walenz A src/utgcns/libcns/unitigConsensus.H nihh20151120Brian P. Walenz A src/utgcns/utgcns.C nihh20151120Brian P. Walenz A src/pipelines/canu/Meryl.pm nihh20151119Sergey Koren A src/pipelines/canu.pl nihh20151119Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20151119Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151119Sergey Koren A src/pipelines/canu/Meryl.pm nihh20151119Sergey Koren A src/pipelines/canu/Defaults.pm nihh20151119Sergey Koren D src/bogart/AS_BAT_PromoteToSingleton.H bogart/AS_BAT_BreakRepeats.H D src/stores/tgStoreCompress.C src/stores/tgStore.C D src/bogart/AS_BAT_FragmentInfo.H src/bogart/AS_BAT_Datatypes.H A src/overlapErrorAdjustment/findErrors-Analyze_Alignment.C nihh20160518Brian P. Walenz A src/utgcns/libcns/NOTES nihh20160518Brian P. Walenz A src/bogart/AS_BAT_MergeUnitigs.C nihh20160517Brian P. Walenz A src/bogart/AS_BAT_MergeUnitigs.H nihh20160517Brian P. Walenz A src/bogart/bogart.C nihh20160517Brian P. Walenz A src/bogart/bogart.mk nihh20160517Brian P. Walenz A src/Makefile nihh20160516Sergey Koren A src/correction/errorEstimate.C nihh20160516Sergey Koren A src/correction/errorEstimate.mk nihh20160516Sergey Koren A src/main.mk nihh20160516Sergey Koren A src/pipelines/canu.pl nihh20160516Sergey Koren A src/pipelines/canu/Defaults.pm nihh20160516Sergey Koren A src/pipelines/canu/ErrorEstimate.pm nihh20160516Sergey Koren A src/pipelines/canu/Execution.pm nihh20160516Brian P. Walenz A src/pipelines/canu/Grid_LSF.pm nihh20160516Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160516Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20160516Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20160516Brian P. Walenz A src/stores/ovStoreBuild.C nihh20160516Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160516Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160513Brian P. Walenz A src/pipelines/canu/Grid_Slurm.pm nihh20160513Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C nihh20160513Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160513Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160513Brian P. Walenz A src/bogart/bogart.C nihh20160513Brian P. Walenz A src/bogart/bogart.mk nihh20160513Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160513Brian P. Walenz A src/bogart/AS_BAT_Outputs.H nihh20160513Brian P. Walenz A src/bogart/bogart.C nihh20160513Brian P. Walenz A src/bogart/buildGraph.C nihh20160513Brian P. Walenz A src/pipelines/canu/Output.pm nihh20160513Brian P. Walenz A documentation/source/faq.rst nihh20160512Sergey Koren A src/bogart/AS_BAT_Unitig.H nihh20160510Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160510Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160510Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160510Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H nihh20160510Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160510Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H nihh20160510Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20160510Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C nihh20160509Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H nihh20160509Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20160509Brian P. Walenz A src/pipelines/canu/OverlapMMap.pm nihh20160509Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20160509Brian P. Walenz A documentation/source/parameter-reference.rst nihh20160506Sergey Koren A documentation/source/conf.py nihh20160506Sergey Koren A documentation/source/faq.rst nihh20160506Sergey Koren A documentation/source/index.rst nihh20160506Sergey Koren A documentation/source/parameter-reference.rst nihh20160506Sergey Koren A documentation/source/commands/bogart.rst nihh20160506Sergey Koren A documentation/source/commands/gatekeeperDumpFASTQ.rst nihh20160506Sergey Koren A documentation/source/commands/ovStoreDump.rst nihh20160506Sergey Koren A documentation/source/index.rst nihh20160506Sergey Koren A documentation/source/parameter-reference.rst nihh20160506Sergey Koren A documentation/source/tutorial.rst nihh20160506Sergey Koren A documentation/source/quick-start.rst nihh20160506Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20160504Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160504Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20160504Brian P. Walenz A src/stores/tgStoreCoverageStat.C nihh20160504Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160504Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160504Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160504Brian P. Walenz A src/stores/tgStoreCoverageStat.C nihh20160503Brian P. Walenz A src/pipelines/canu.pl nihh20160502Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Correct_Frags.C nihh20160502Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Prefix_Edit_Distance.C nihh20160502Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Read_Olaps.C nihh20160502Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps-Redo_Olaps.C nihh20160502Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps.C nihh20160502Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps.H nihh20160502Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20160502Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20160502Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20160502Brian P. Walenz A src/pipelines/canu/OverlapInCore.pm nihh20160502Brian P. Walenz A src/pipelines/canu/OverlapMMap.pm nihh20160502Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20160502Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20160502Brian P. Walenz A src/pipelines/canu/OverlapMMap.pm nihh20160502Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20160502Brian P. Walenz A README.md nihh20160428Brian P. Walenz A README.md nihh20160428Brian P. Walenz A src/stores/ovStoreDump.C nihh20160428Brian P. Walenz A src/stores/ovStoreDump.C nihh20160428Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20160428Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20160428Brian P. Walenz A src/overlapErrorAdjustment/correctOverlaps.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.H nihh20160428Brian P. Walenz A src/bogart/bogart.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H nihh20160428Brian P. Walenz A addCopyrights.dat nihh20160428Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160428Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.H nihh20160428Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.H nihh20160428Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_Joining.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H nihh20160428Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H nihh20160428Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.H nihh20160428Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.H nihh20160428Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_PromoteToSingleton.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.H nihh20160428Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_Unitig.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160428Brian P. Walenz A src/bogart/AS_BAT_UnitigVector.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_UnitigVector.H nihh20160428Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddFrag.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C nihh20160428Brian P. Walenz A src/bogart/bogart.C nihh20160428Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160427Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160427Brian P. Walenz A src/bogart/AS_BAT_Outputs.H nihh20160427Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C nihh20160427Brian P. Walenz A src/bogart/bogart.C nihh20160427Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20160427Sergey Koren A src/AS_UTL/AS_UTL_stackTrace.C nihh20160426Sergey Koren A src/Makefile nihh20160426Sergey Koren A src/bogart/AS_BAT_OverlapCache.C nihh20160426Sergey Koren A src/falcon_sense/falcon_sense.C nihh20160426Sergey Koren A src/pipelines/canu/Defaults.pm nihh20160426Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20160426Sergey Koren A src/utgcns/utgcns.C nihh20160426Sergey Koren A documentation/source/quick-start.rst nihh20160426Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160413Brian P. Walenz A src/AS_global.H nihh20160426Brian P. Walenz A src/pipelines/canu.pl nihh20160426Brian P. Walenz A src/AS_UTL/stddev.H nihh20160426Brian P. Walenz A src/AS_UTL/stddev.H nihh20160426Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20160426Sergey Koren A documentation/source/parameter-reference.rst nihh20160426Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160426Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160426Brian P. Walenz A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20160426Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160426Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160426Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20160426Brian P. Walenz A src/stores/gkStore.C nihh20160426Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160426Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160426Brian P. Walenz A src/pipelines/canu/Grid_LSF.pm nihh20160426Brian P. Walenz A src/pipelines/canu/Grid_PBSTorque.pm nihh20160426Brian P. Walenz A src/pipelines/canu/Grid_SGE.pm nihh20160426Brian P. Walenz A src/pipelines/canu/Grid_Slurm.pm nihh20160426Brian P. Walenz A src/pipelines/canu/OverlapBasedTrimming.pm nihh20160426Brian P. Walenz A src/pipelines/canu.pl nihh20160426Brian P. Walenz A src/stores/gatekeeperDumpFASTQ.C nihh20160426Brian P. Walenz A documentation/source/quick-start.rst nihh20160425Brian P. Walenz A src/pipelines/canu/Defaults.pm nihh20160422Sergey Koren A src/pipelines/canu/Unitig.pm nihh20160422Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20160422Sergey Koren A src/pipelines/canu/Defaults.pm nihh20160422Sergey Koren A src/pipelines/canu/OverlapMMap.pm nihh20160422Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20160422Sergey Koren A src/pipelines/canu.pl nihh20160422Sergey Koren A src/pipelines/canu/Defaults.pm nihh20160422Sergey Koren A src/pipelines/canu/Unitig.pm nihh20160422Sergey Koren A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160422Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160422Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160422Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160422Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.H nihh20160422Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C nihh20160422Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H nihh20160422Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160422Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.H nihh20160422Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C nihh20160422Brian P. Walenz A src/bogart/AS_BAT_Unitig.C nihh20160422Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160422Brian P. Walenz A src/bogart/AS_BAT_UnitigVector.C nihh20160422Brian P. Walenz A src/bogart/AS_BAT_UnitigVector.H nihh20160422Brian P. Walenz A src/bogart/bogart.C nihh20160422Brian P. Walenz A src/pipelines/canu/OverlapStore.pm nihh20160422Sergey Koren A src/pipelines/canu/HTML.pm nihh20160420Brian P. Walenz A src/pipelines/canu/Execution.pm nihh20160420Sergey Koren A src/overlapInCore/overlapPair.C nihh20160420Sergey Koren A src/stores/ovStoreStats.C nihh20160420Sergey Koren A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160420Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160419Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160419Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160419Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_Logging.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_Logging.H nihh20160418Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C nihh20160418Brian P. Walenz A src/bogart/bogart.C nihh20160418Brian P. Walenz A src/stores/ovStoreDump.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160418Brian P. Walenz A addCopyrights.dat nihh20160418Brian P. Walenz A src/main.mk nihh20160418Brian P. Walenz A src/stores/tgStoreCompress.C nihh20160418Brian P. Walenz A src/stores/tgStoreCompress.mk nihh20160418Brian P. Walenz A src/pipelines/canu/CorrectReads.txt nihh20160418Brian P. Walenz A src/pipelines/canu/Execution.txt nihh20160418Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160418Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.txt nihh20160418Brian P. Walenz A src/AS_UTL/stddevTest.C nihh20160418Brian P. Walenz A documentation/source/quick-start.rst nihh20160415Sergey Koren A src/pipelines/canu/Unitig.pm nihh20160413Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160413Brian P. Walenz A addCopyrights.dat nihh20160413Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160413Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.H nihh20160413Brian P. Walenz A src/bogart/AS_BAT_PromoteToSingleton.H nihh20160413Brian P. Walenz A src/bogart/bogart.C nihh20160413Brian P. Walenz A src/bogart/bogart.mk nihh20160413Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160413Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160413Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160413Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160413Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160413Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160413Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H nihh20160413Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160413Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C nihh20160413Brian P. Walenz A src/bogart/AS_BAT_SetParentAndHang.C nihh20160413Brian P. Walenz A src/bogart/bogart.mk nihh20160413Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160412Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160412Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160412Brian P. Walenz A src/bogart/bogart.C nihh20160412Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160412Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160411Sergey Koren A src/stores/tgStoreLoad.C nihh20160411Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160408Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160408Brian P. Walenz A src/bogart/AS_BAT_Unitig.C nihh20160408Brian P. Walenz A src/bogart/bogart.C nihh20160408Brian P. Walenz A src/pipelines/canu/Configure.pm nihh20160408Sergey Koren A src/pipelines/canu/OverlapMMap.pm nihh20160408Sergey Koren A src/pipelines/canu/Execution.pm nihh20160408Sergey Koren A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160408Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C nihh20160407Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C nihh20160407Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160407Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H nihh20160407Brian P. Walenz A src/Makefile nihh20160406Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160406Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160406Brian P. Walenz A src/bogart/AS_BAT_Logging.C nihh20160406Brian P. Walenz A src/bogart/AS_BAT_Logging.H nihh20160406Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160406Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.H nihh20160406Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C nihh20160406Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H nihh20160406Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C nihh20160406Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160406Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C nihh20160406Brian P. Walenz A src/bogart/AS_BAT_Unitig.C nihh20160406Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160406Brian P. Walenz A src/bogart/AS_BAT_UnitigVector.C nihh20160406Brian P. Walenz A src/bogart/AS_BAT_UnitigVector.H nihh20160406Brian P. Walenz A src/bogart/bogart.C nihh20160406Brian P. Walenz A src/bogart/bogart.mk nihh20160406Brian P. Walenz A src/AS_UTL/stddev.H nihh20160406Brian P. Walenz A src/AS_UTL/stddev.H nihh20160331Sergey Koren A src/stores/ovStoreStats.C nihh20160331Sergey Koren A src/pipelines/canu/Defaults.pm nihh20160330Sergey Koren A src/pipelines/canu.pl nihh20160330Sergey Koren A src/bogart/AS_BAT_Outputs.C nihh20160330Sergey Koren A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20160330Sergey Koren A src/overlapErrorAdjustment/correctOverlaps.C nihh20160330Sergey Koren A src/pipelines/canu/Configure.pm nihh20160330Sergey Koren A src/AS_UTL/stddev.H nihh20160330Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160330Brian P. Walenz A src/main.mk nihh20160330Brian P. Walenz A src/pipelines/canu/Unitig.pm nihh20160329Sergey Koren A src/overlapInCore/overlapInCore-Process_String_Overlaps.C nihh20160323Sergey Koren A src/overlapInCore/overlapInCore.C nihh20160323Sergey Koren A src/overlapInCore/overlapInCore.H nihh20160323Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20160327Sergey Koren A src/pipelines/canu.pl nihh20160327Sergey Koren A src/pipelines/canu/Configure.pm nihh20160327Sergey Koren A src/pipelines/canu/OverlapMhap.pm nihh20160327Sergey Koren A src/pipelines/canu/OverlapErrorAdjustment.pm nihh20160327Sergey Koren A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_Unitig.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160324Brian P. Walenz A src/bogart/AS_BAT_Unitig.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160324Brian P. Walenz A src/bogart/AS_BAT_IntersectBubble.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_PromoteToSingleton.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_Unitig.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160324Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_Unitig.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_Unitig.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160324Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H nihh20160324Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C nihh20160324Brian P. Walenz A src/bogart/bogart.C nihh20160324Brian P. Walenz A src/bogart/AS_BAT_ChunkGraph.C nihh20160324Brian P. Walenz A src/falcon_sense/falcon_sense.C nihh20160324Sergey Koren A src/falcon_sense/falcon_sense.C nihh20160324Sergey Koren A src/falcon_sense/libfalcon/falcon.C nihh20160324Sergey Koren A src/falcon_sense/libfalcon/falcon.H nihh20160324Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20160324Sergey Koren A src/pipelines/canu/Gatekeeper.pm nihh20160324Sergey Koren A src/pipelines/canu/OverlapStore.pm nihh20160323Sergey Koren A src/falcon_sense/libfalcon/falcon.C nihh20160323Sergey Koren A src/overlapInCore/overlapInCore-Process_String_Overlaps.C nihh20160323Sergey Koren A src/overlapInCore/overlapInCore.C nihh20160323Sergey Koren A src/overlapInCore/overlapInCore.H nihh20160323Sergey Koren A src/overlapErrorAdjustment/findErrors-Analyze_Alignment.C nihh20160322Sergey Koren A src/falcon_sense/libfalcon/falcon.C nihh20160322Sergey Koren A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160322Sergey Koren A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160322Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160322Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H nihh20160322Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.C nihh20160322Brian P. Walenz A src/bogart/AS_BAT_PlaceContains.H nihh20160322Brian P. Walenz A src/bogart/AS_BAT_PlaceFragUsingOverlaps.C nihh20160322Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160322Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C nihh20160322Brian P. Walenz A src/bogart/AS_BAT_ReconstructRepeats.C nihh20160322Brian P. Walenz A src/bogart/AS_BAT_SplitDiscontinuous.C nihh20160322Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160322Brian P. Walenz A src/bogart/AS_BAT_Unitig_AddFrag.C nihh20160322Brian P. Walenz A src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C nihh20160322Brian P. Walenz A src/bogart/bogart.C nihh20160322Brian P. Walenz A src/bogart/bogart.mk nihh20160322Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H nihh20160321Brian P. Walenz A src/bogart/AS_BAT_FragmentInfo.C nihh20160321Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160321Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160321Brian P. Walenz A src/bogart/AS_BAT_Outputs.C nihh20160317Brian P. Walenz A src/AS_UTL/stddev.H nihh20160317Brian P. Walenz A src/AS_UTL/stddev.H nihh20160317Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160317Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160317Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160316Brian P. Walenz A src/bogart/bogart.C nihh20160316Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160316Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160316Brian P. Walenz A src/bogart/AS_BAT_Datatypes.H nihh20160316Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C nihh20160316Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.H nihh20160316Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C nihh20160316Brian P. Walenz A src/bogart/AS_BAT_Logging.C nihh20160316Brian P. Walenz A src/AS_UTL/memoryMappedFile.H nihh20160316Brian P. Walenz A src/pipelines/canu/OverlapMhap.pm nihh20160315Sergey Koren A src/bogart/bogart.C nihh20160315Sergey Koren A src/pipelines/canu/Unitig.pm nihh20160315Sergey Koren A src/AS_UTL/memoryMappedFile.H nihh20160315Sergey Koren A src/pipelines/canu.pl nihh20160315Sergey Koren A src/AS_UTL/memoryMappedFile.H nihh20160313Brian P. Walenz A src/stores/ovOverlap.C nihh20160311Sergey Koren A src/stores/ovStore.H nihh20160311Sergey Koren A src/stores/ovStoreDump.C nihh20160311Sergey Koren A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160311Sergey Koren A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160311Sergey Koren A src/bogart/AS_BAT_MergeSplitJoin.C nihh20160311Sergey Koren A src/bogart/AS_BAT_MergeSplitJoin.H nihh20160311Sergey Koren A src/bogart/bogart.C nihh20160311Sergey Koren A src/bogart/bogart.mk nihh20160311Sergey Koren A src/stores/ovOverlap.C nihh20160311Sergey Koren A src/stores/ovStore.H nihh20160311Sergey Koren A src/stores/ovStoreDump.C nihh20160311Sergey Koren A src/pipelines/canu.pl nihh20160311Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.C nihh20160311Brian P. Walenz A src/bogart/AS_BAT_BestOverlapGraph.H nihh20160311Brian P. Walenz A src/bogart/AS_BAT_BreakRepeats.C nihh20160311Brian P. Walenz A src/bogart/AS_BAT_BreakRepeats.H nihh20160311Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.C nihh20160311Brian P. Walenz A src/bogart/AS_BAT_MarkRepeatReads.H nihh20160311Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.C nihh20160311Brian P. Walenz A src/bogart/AS_BAT_PopBubbles.H nihh20160311Brian P. Walenz A src/bogart/bogart.C nihh20160311Brian P. Walenz A src/bogart/bogart.mk nihh20160311Brian P. Walenz A src/pipelines/bogart-sweep.pl nihh20160310Brian P. Walenz A src/AS_UTL/intervalListTest.C nihh20160310Brian P. Walenz A src/AS_UTL/stddev.H nihh20160310Brian P. Walenz A src/AS_UTL/stddevTest.C nihh20160310Brian P. Walenz A src/bogart/AS_BAT_PopulateUnitig.C nihh20160310Brian P. Walenz A src/bogart/AS_BAT_OverlapCache.C nihh20160310Brian P. Walenz A src/bogart/AS_BAT_Unitig.H nihh20160310Brian P. Walenz A src/bogart/bogart.C nihh20160310Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160310Brian P. Walenz A src/bogart/AS_BAT_Instrumentation.C nihh20160310Brian P. Walenz A src/stores/ovStore.H nihh20160310Brian P. Walenz A src/AS_UTL/intervalList.H nihh20160310Brian P. Walenz A src/AS_UTL/intervalList.H nihh20160310Brian P. Walenz A src/AS_UTL/memoryMappedFile.H nihh20160310Brian P. Walenz A documentation/source/quick-start.rst nihh20160309Sergey Koren A src/pipelines/canu/CorrectReads.pm nihh20160309Sergey Koren A documentation/source/quick-start.rst nihh20160307Sergey Koren canu-1.3/addCopyrights.pl000066400000000000000000000276371272335422300154660ustar00rootroot00000000000000#!/usr/local/bin/perl use strict; my @dateStrings = ( "???", "JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC" ); # If set, rename original files to name.ORIG, rewrite files with updated copyright text. # If not, create new name.MODIFIED files with updated copyright text. # my $doForReal = 1; # # The change data 'addCopyrights.dat' contains lines of two types: # # A -- a committed change to this file # D -- 'file' used to be called 'oldfile' # # The 'D' lines DO NOT map existing A lines. They just emit 'this file derived from' lines in the # copyright text. # # When a file is added to the repo, nothing special needs to be done; an 'A' line # is emitted for the new file. If a file comes about because of a copy (of an existing file), # a 'D' line needs to be added. # # If a file is renamed, a 'D' line needs to be added, and all previous mentions # of the original name need to be updated to the new name. # sub toList (@) { my @all = sort { $a <=> $b } @_; my $ret; $ret = substr($all[0], 0, 4); shift @all; foreach my $a (@all) { $a = substr($a, 0, 4); if ($ret =~ /^(\d+)$/) { if ($1 == $a) { } elsif ($1 + 1 == $a) { $ret = "$1-$a"; } else { $ret = "$1,$a"; } } if ($ret =~ /^(.*)-(\d+)$/) { if ($2 == $a) { } elsif ($2 + 1 == $a) { $ret = "$1-$a"; } else { $ret = "$1-$2,$a"; } } if ($ret =~ /^(.*),(\d+)$/) { if ($2 == $a) { } elsif ($2 + 1 == $a) { $ret = "$1,$2-$a"; } else { $ret = "$1,$2,$a"; } } } return($ret); } sub splitAC ($@) { my $cc = shift @_; my @AC = @_; my @AClist; my %dates; foreach my $ac (@AC) { if ($ac =~ m/^(....)(\d\d\d\d\d\d\d\d)(.*)$/) { $dates{"$1$3"} .= "$2\n"; } else { die "$ac failed\n"; } } foreach my $ac (keys %dates) { my @dates = split '\n', $dates{$ac}; @dates = sort { $a <=> $b } @dates; my $years = toList(@dates); my $ord = $dates[0]; my $bgn = $dates[0]; my $end = $dates[ scalar(@dates)-1 ]; if ($bgn =~ m/^(\d\d\d\d)(\d\d)(\d\d)$/) { $bgn = "$1-$dateStrings[$2]-$3"; } else { die "bgn date $bgn\n"; } if ($end =~ m/^(\d\d\d\d)(\d\d)(\d\d)$/) { $end = "$1-$dateStrings[$2]-$3"; } else { die "bgn date $end\n"; } my $org; my $nam; if ($ac =~ m/^(....)(.*)$/) { $org = $1; $nam = $2; } else { die "$ac match\n"; } my $dates = "from $bgn to $end"; if ($bgn eq $end) { $dates = "on $bgn"; } if ($org eq "nihh") { $dates = "beginning on $bgn"; } my $str; if ($org eq "craa") { $str .= " $cc $nam $dates\n"; $str .= " $cc are Copyright $years Applera Corporation, and\n"; $str .= " $cc are subject to the GNU General Public License version 2\n"; $str .= " $cc\n"; } elsif ($org eq "tigr") { $str .= " $cc $nam $dates\n"; $str .= " $cc are Copyright $years The Institute for Genomics Research, and\n"; $str .= " $cc are subject to the GNU General Public License version 2\n"; $str .= " $cc\n"; } elsif ($org eq "jcvi") { $str .= " $cc $nam $dates\n"; $str .= " $cc are Copyright $years J. Craig Venter Institute, and\n"; $str .= " $cc are subject to the GNU General Public License version 2\n"; $str .= " $cc\n"; } elsif ($org eq "bnbi") { $str .= " $cc $nam $dates\n"; $str .= " $cc are Copyright $years Battelle National Biodefense Institute, and\n"; $str .= " $cc are subject to the BSD 3-Clause License\n"; $str .= " $cc\n"; } elsif ($org eq "nihh") { $str .= " $cc $nam $dates\n"; $str .= " $cc are a 'United States Government Work', and\n"; $str .= " $cc are released in the public domain\n"; $str .= " $cc\n"; } elsif ($org eq "none") { $str .= " $cc $nam $dates\n"; $str .= " $cc are Copyright $years $nam, and\n"; $str .= " $cc are subject to the GNU General Public License version 2\n"; $str .= " $cc\n"; } else { die "$ac org\n"; } push @AClist, "$ord\0$str"; } @AClist = sort { $a <=> $b } @AClist; foreach my $a (@AClist) { (undef, $a) = split '\0', $a; } return(@AClist); } # Load the previously generated change data my %authcopy; my %derived; { open(F, "< addCopyrights.dat"); while () { chomp; if (m/^A\s+(\S+)\s+(.*)$/) { $authcopy{$1} .= "$2\n"; } elsif (m/^D\s+(\S+)\s+(\S+)$/) { $derived{$1} .= "$2\n"; } else { die "invalid addCopyrights.dat line '$_'\n"; } } close(F); } # Process each file. open(FIN, "find kmer src -type f -print |") or die "Failed to launch 'find'\n"; #open(OUT, "> addCopyrights.dat.new") or die "Failed to open 'addCopyrights.dat.new' for writing: $!\n"; while () { chomp; my $file = $_; $file = $1 if ($_ =~ m/^\.\/(.*)$/); # Remove leading ./ added by find. my @lines; next if ($file =~ m/\.mk$/); next if ($file =~ m/Makefile/); next if ($file =~ m/\.sh$/); next if ($file =~ m/\.py$/); next if ($file =~ m/\.jar$/); next if ($file =~ m/\.tar$/); next if ($file =~ m/\.bin$/); # falcon_sense next if ($file =~ m/\.gz$/); next if ($file =~ m/\.json$/); next if ($file =~ m/\.json.README$/); next if ($file =~ m/\.css$/); next if ($file =~ m/\.fasta$/); # meryl test next if ($file =~ m/\.dat$/); # src/overlapInCore/liboverlap/prefixEditDistance-matchLimitData/prefixEditDistance-matchLimit-*.dat next if ($file =~ m/md5/); next if ($file =~ m/mt19937ar/); next if ($file =~ m/\.jpg$/); next if ($file =~ m/README/); next if ($file =~ m/\.dat$/); next if ($file =~ m/libboost/); my $cb = "/"; my $cc = "*"; my $ce = "/"; if ($file =~ m/\.p[lm]$/) { $cb = "#"; $cc = "#"; $ce = "#"; } my $iskmer = 0; $iskmer = 1 if ($file =~ m/^kmer/); $iskmer = 1 if ($file =~ m/meryl/); $iskmer = 1 if ($file =~ m/bitEncodings/); $iskmer = 1 if ($file =~ m/bitOperations/); $iskmer = 1 if ($file =~ m/bitPackedArray/); $iskmer = 1 if ($file =~ m/bitPackedFile/); $iskmer = 1 if ($file =~ m/bitPacking/); $iskmer = 1 if ($file =~ m/decodeBooleanString/); $iskmer = 1 if ($file =~ m/dnaAlphabets/); $iskmer = 1 if ($file =~ m/intervalList/); $iskmer = 1 if ($file =~ m/kMer/); $iskmer = 1 if ($file =~ m/kMerHuge/); $iskmer = 1 if ($file =~ m/kMerTiny/); $iskmer = 1 if ($file =~ m/memoryMappedFile/); $iskmer = 1 if ($file =~ m/memoryMappedFileTest/); $iskmer = 1 if ($file =~ m/readBuffer/); $iskmer = 1 if ($file =~ m/speedCounter/); $iskmer = 1 if ($file =~ m/splitToWords/); $iskmer = 1 if ($file =~ m/sweatShop/); $iskmer = 1 if ($file =~ m/testHashTable/); $iskmer = 1 if ($file =~ m/testRand/); $iskmer = 1 if ($file =~ m/testVar/); $iskmer = 1 if ($file =~ m/timeAndSize/); if (($file !~ m/\.[CHch]$/) && ($file !~ m/\.p[lm]/)) { print STDERR "Won't process: '$file'\n"; next; } my @AC = split '\n', $authcopy{$file}; #foreach my $ac (@AC) { # print OUT "A\t$file\t$ac\n"; #} my @AClist = splitAC($cc, @AC); my %DElist; my @DElist = split '\n', $derived{$file}; foreach my $d (@DElist) { next if ($d eq ""); next if (lc $d eq lc $file); $DElist{$d}++; } undef @DElist; if (scalar(keys %DElist) > 0) { foreach my $d (keys %DElist) { push @DElist, " $cc $d\n"; } @DElist = sort @DElist; #foreach my $d (@DElist) { # if ($d =~ m/^\s*\S\s*(\S+)$/) { # print OUT "D\t$file\t$1\n"; # } else { # die "Failed to match DElist line '$d'\n"; # } #} unshift @DElist, " $cc\n"; unshift @DElist, " $cc This file is derived from:\n"; push @DElist, " $cc\n"; } if ($file =~ m/\.pl$/) { push @lines, "#!/usr/bin/env perl\n"; } push @lines, "\n"; push @lines, "$cb" . $cc x 78 . "\n"; push @lines, " $cc\n"; push @lines, " $cc This file is part of canu, a software program that assembles whole-genome\n"; push @lines, " $cc sequencing reads into contigs.\n"; push @lines, " $cc\n"; push @lines, " $cc This software is based on:\n"; #push @lines, " $cc RELEASE_1-3_2004-03-17 of the 'Celera Assembler' (http://wgs-assembler.sourceforge.net)\n"; push @lines, " $cc 'Celera Assembler' (http://wgs-assembler.sourceforge.net)\n"; push @lines, " $cc the 'kmer package' (http://kmer.sourceforge.net)\n"; push @lines, " $cc both originally distributed by Applera Corporation under the GNU General\n"; push @lines, " $cc Public License, version 2.\n"; push @lines, " $cc\n"; push @lines, " $cc Canu branched from Celera Assembler at its revision 4587.\n"; push @lines, " $cc Canu branched from the kmer project at its revision 1994.\n"; push @lines, " $cc\n"; push @lines, @DElist; push @lines, " $cc Modifications by:\n"; push @lines, " $cc\n"; push @lines, @AClist; push @lines, " $cc File 'README.licenses' in the root directory of this distribution contains\n"; push @lines, " $cc full conditions and disclaimers for each license.\n"; push @lines, " $cc$ce\n"; push @lines, "\n"; my $start = 1; # To skip comment lines at the start of the file (the previous copyright block). open(F, "< $file") or die "Failed to open '$file' for reading: $!\n"; while () { s/\s+$//; # Remove trailing spaces because they bug me. # If a single "/*" at the start of the line, assume this is NOT an old copyright block, # but a copyright block (or just a comment) from some third party code. if ($_ eq "\/\*") { print STDERR "Foreign code found: '$file'\n"; $start = 0; } # If not at the start, add the line. if ($start == 0) { push @lines, "$_\n"; next; } # Else, we're at the start; if blank or a comment, skip it. if (($_ eq "") || # Blank lines ($_ =~ m/^[\/\s]\*/) || # C-style comment (the old copyright block) ($_ =~ m/^\s*##/) || # Perl comment, at least two #'s ($_ =~ m/^\s*#$/) || # Perl comment, exactly one # ($_ =~ m/^\s*#\s/) || # Perl comment, a single # followed by a space (so we don't catch #! lines) ($_ =~ m/^\s*#\!/)) { # #! line. I guess we do want to skip them now next; } # Else, add the line, and declare that we're no longer at the start. push @lines, "$_\n"; $start = 0; } close(F); if ($doForReal) { rename "$file", "$file.ORIG"; open(F, "> $file") or die "Failed to open '$file' for writing: $!\n"; print F @lines; close(F); } else { open(F, "> $file.MODIFIED") or die "Failed to open '$file.MODIFIED' for writing: $!\n"; print F @lines; close(F); } } close(FIN); canu-1.3/documentation/000077500000000000000000000000001272335422300151575ustar00rootroot00000000000000canu-1.3/documentation/Makefile000066400000000000000000000151721272335422300166250ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source .PHONY: all help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext all: html help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/canu.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/canu.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/canu" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/canu" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." canu-1.3/documentation/source/000077500000000000000000000000001272335422300164575ustar00rootroot00000000000000canu-1.3/documentation/source/command-reference.rst000066400000000000000000000127021272335422300225650ustar00rootroot00000000000000 .. _command-reference: Canu Command Reference ====================== Every command, even the useless ones. Commands marked as 'just usage' were automagically generated from the command line usage summary. Yes, some of them even crashed. :doc:`commands/bogart` (just usage) The unitig construction algorithm. BOG stands for Best Overlap Graph; we haven't figured out what ART stands for. :doc:`commands/bogus` (just usage) A unitig construction algorithm simulator. Given reads mapped to a reference, returns the largest unitigs possible. :doc:`commands/canu` (just usage) The executive in charge! Coordinates all these commands to make an assembler. :doc:`commands/correctOverlaps` (just usage) Part of Overlap Error Adjustment, recomputes overlaps given a set of read corrections. :doc:`commands/estimate-mer-threshold` (just usage) Decides on a k-mer threshold for overlapInCore seeds. :doc:`commands/fastqAnalyze` (just usage) Analyzes a FASTQ file and reports the best guess of the QV encoding. Can also rewrite the FASTQ to be in Sanger QV format. :doc:`commands/fastqSample` (just usage) Extracts random reads from a single or mated FASTQ file. Extracts based on desired coverage, desired number of reads/pairs, desired fraction of the total, or desired total length. :doc:`commands/fastqSimulate` (just usage) Creates reads with unbiased errors from a FASTA sequence. :doc:`commands/fastqSimulate-sort` (just usage) Given input from fastqSimulate, sorts the reads by position in the reference. :doc:`commands/filterCorrectionOverlaps` (just usage) Part of Read Correction, filters overlaps that shouldn't be used for correcting reads. :doc:`commands/findErrors` (just usage) Part of Overlap Error Adjustment, generates a multialignment for each read, outputs a list of suspected errors in the read. :doc:`commands/gatekeeperCreate` (just usage) Loads FASTA or FASTQ reads into the canu read database, gkpStore. :doc:`commands/gatekeeperDumpFASTQ` (just usage) Outputs FASTQ reads fromt the canu read database, gkpStore. :doc:`commands/gatekeeperDumpMetaData` (just usage) Outputs read and library metadata fromt the canu read database, gkpStore. :doc:`commands/gatekeeperPartition` (just usage) Part of Consensus, rearranges the canu read database, gkpStore, to localize read to unitigs. :doc:`commands/generateCorrectionLayouts` (just usage) Part of Read Correction, generates the multialignment layout used to correct reads. :doc:`commands/leaff` (just usage) Not actually part of canu, but it came along with meryl. Provides random access to FASTA, FASTQ and gkpStore. Also does some analysis tasks. Handy Swiss Army knife type of tool. :doc:`commands/meryl` (just usage) Counts k-mer occurrences in FASTA, FASTQ and gkpStore. Performs mathematical and logical operations on the resulting k-mer databases. :doc:`commands/mhapConvert` (just usage) Convert mhap output to overlap output. :doc:`commands/ovStoreBucketizer` (just usage) Part of the parallel overlap store building pipeline, loads raw overlaps from overlapper into the store. :doc:`commands/ovStoreBuild` (just usage) Sequentially builds an overlap store from raw overlaps. Simplest to run, but slow on large datasets. :doc:`commands/ovStoreDump` (just usage) Dumps overlaps from the overlap store, ovlStore. :doc:`commands/ovStoreIndexer` (just usage) Part of the parallel overlap store building pipeline, finalizes the store, after sorting with ovStoreSorter. :doc:`commands/ovStoreSorter` (just usage) Part of the parallel overlap store building pipeline, sorts overlaps loaded into the store by ovStoreBucketizer. :doc:`commands/overlapConvert` (just usage) Reads raw overlapper output, writes overlaps as ASCII. The reverse of overlapImport. :doc:`commands/overlapImport` (just usage) Reads ASCII overlaps in a few different formats, writes either 'raw overlapper output' or creates an ovlStore. :doc:`commands/overlapInCore` (just usage) The classic overlapper algorithm. :doc:`commands/overlapInCorePartition` (just usage) Generate partitioning to run overlapInCore jobs in parallel. :doc:`commands/overlapPair` (just usage) An *experimental* algorithm to recompute overlaps and output the alignments. :doc:`commands/prefixEditDistance-matchLimitGenerate` (just usage) Generate source code files with data representing the minimum length of a good overlap given some number of errors. :doc:`commands/splitReads` (just usage) Part of Overlap Based Trimming, splits reads based on overlaps, specifically, looking for PacBio hairpin adapter signatures. :doc:`commands/tgStoreCoverageStat` (just usage) Analyzes tigs in the tigStore, computes the classic `arrival rate statistic `_. :doc:`commands/tgStoreDump` (just usage) Analyzes and outputs tigs from the tigStore, in various formats (FASTQ, layouts, multialignments, etc). :doc:`commands/tgStoreFilter` (just usage) Analyzes tigs in the tigStore, marks those that appear to be spurious 'degenerate' tigs. :doc:`commands/tgStoreLoad` (just usage) Loads tigs into a tigStore. :doc:`commands/tgTigDisplay` (just usage) Displays the tig contained in a binary multialignment file, as output by utgcns. :doc:`commands/trimReads` (just usage) Part of Overlap Based Trimming, trims reads based on overlaps. :doc:`commands/utgcns` (just usage) Generates a multialignment for a tig, based on the layout stored in tigStore. Outputs FASTQ, layouts and binary mutlialignment files. canu-1.3/documentation/source/commands/000077500000000000000000000000001272335422300202605ustar00rootroot00000000000000canu-1.3/documentation/source/commands/bogart.rst000066400000000000000000000046631272335422300223010ustar00rootroot00000000000000bogart ~~~~~~ :: usage: bogart -o outputName -O ovlStore -G gkpStore -T tigStore -O Mandatory path to an ovlStore. -G Mandatory path to a gkpStore. -T Mandatory path to a tigStore (can exist or not). -o prefix Mandatory name for the output files -B b Target number of fragments per tigStore (consensus) partition Algorithm Options -gs Genome size in bases. -J Join promiscuous unitigs using unused best edges. -SR Shatter repeats, don't rebuild. -R Shatter repeats (-SR), then rebuild them -RL len Force reads below 'len' bases to be singletons. This WILL cause CGW to fail; diagnostic only. -threads N Use N compute threads during repeat detection. 0 - use OpenMP default (default) 1 - use one thread Overlap Selection - an overlap will be considered for use in a unitig under the following conditions: When constructing the Best Overlap Graph and Promiscuous Unitigs ('g'raph): -eg 0.020 no more than 0.020 fraction (2.0%) error ** DEPRECATED ** When loading overlaps, an inflated maximum (to allow reruns with different error rates): -eM 0.05 no more than 0.05 fraction (5.0%) error in any overlap loaded into bogart the maximum used will ALWAYS be at leeast the maximum of the four error rates For all, the lower limit on overlap length -el 500 no shorter than 40 bases Overlap Storage -M gb Use at most 'gb' gigabytes of memory for storing overlaps. -N num Load at most 'num' overlaps per read. -create Only create the overlap graph, save to disk and quit. -save Save the overlap graph to disk, and continue. Debugging and Logging -D enable logging/debugging for a specific component. -d disable logging/debugging for a specific component. overlapScoring allBestEdges chunkGraph buildUnitig placeUnplaced bubbles splitDiscontinuous intermediateUnitigs setParentAndHang stderr No output prefix name (-o option) supplied. No gatekeeper store (-G option) supplied. No overlap store (-O option) supplied. No output tigStore (-T option) supplied. canu-1.3/documentation/source/commands/bogus.rst000066400000000000000000000002671272335422300221360ustar00rootroot00000000000000bogus ~~~~~~ :: ERROR: No input matches supplied (either -nucmer or -snapper). ERROR: No reference sequence supplied (-reference). ERROR: No output prefix supplied (-output). canu-1.3/documentation/source/commands/canu.rst000066400000000000000000000035171272335422300217460ustar00rootroot00000000000000canu ~~~~~~ :: -- Detected Java(TM) Runtime Environment '1.8.0_60' (from 'java'). usage: canu [-correct | -trim | -assemble] \ [-s ] \ -p \ -d \ genomeSize=[g|m|k] \ errorRate=0.X \ [other-options] \ [-pacbio-raw | -pacbio-corrected | -nanopore-raw | -nanopore-corrected] *fastq By default, all three stages (correct, trim, assemble) are computed. To compute only a single stage, use: -correct - generate corrected reads -trim - generate trimmed reads -assemble - generate an assembly The assembly is computed in the (created) -d , with most files named using the -p . The genome size is your best guess of the genome size of what is being assembled. It is used mostly to compute coverage in reads. Fractional values are allowed: '4.7m' is the same as '4700k' and '4700000' The errorRate is not used correctly (we're working on it). Don't set it If you want to change the defaults, use the various utg*ErrorRate options. A full list of options can be printed with '-options'. All options can be supplied in an optional sepc file. Reads can be either FASTA or FASTQ format, uncompressed, or compressed with gz, bz2 or xz. Reads are specified by the technology they were generated with: -pacbio-raw -pacbio-corrected -nanopore-raw -nanopore-corrected Complete documentation at http://canu.readthedocs.org/en/latest/ ERROR: Assembly name prefix not supplied with -p. ERROR: Directory not supplied with -d. ERROR: Required parameter 'genomeSize' is not set canu-1.3/documentation/source/commands/correctOverlaps.rst000066400000000000000000000020451272335422300241700ustar00rootroot00000000000000correctOverlaps ~~~~~~ :: ERROR: no input gatekeeper store (-G) supplied. ERROR: no input overlap store (-O) supplied. ERROR: no input read corrections file (-c) supplied. ERROR: no output erates file (-o) supplied. USAGE: correctOverlaps [-d ] [-o ] [-q ] [-x ] [-F OlapFile] [-S OlapStore] [-c ] [-e Recalculates overlaps for frags .. in using corrections in Options: -e specifies binary file to dump corrected erates to for later updating of olap store by update-erates -F specify file of sorted overlaps to use (in the format produced by get-olaps -o specifies name of file to which OVL messages go -q overlaps less than this error rate are automatically output -S specify the binary overlap store containing overlaps to use canu-1.3/documentation/source/commands/createFalconSenseInputs.rst000066400000000000000000000000451272335422300256000ustar00rootroot00000000000000createFalconSenseInputs ~~~~~~ :: canu-1.3/documentation/source/commands/erateEstimate.rst000066400000000000000000000000551272335422300236060ustar00rootroot00000000000000erateEstimate ~~~~~~ :: Opening '(null)' canu-1.3/documentation/source/commands/estimate-mer-threshold.rst000066400000000000000000000001661272335422300254030ustar00rootroot00000000000000estimate-mer-threshold ~~~~~~ :: usage: estimate-mer-threshold -m mercounts -m mercounts file of mercounts canu-1.3/documentation/source/commands/fastqAnalyze.rst000066400000000000000000000013221272335422300234520ustar00rootroot00000000000000fastqAnalyze ~~~~~~ :: usage: fastqAnalyze [-stats] [-o output.fastq] input.fastq If no options are given, input.fastq is analyzed and a best guess for the QV encoding is output. Otherwise, the QV encoding is converted to Sanger-style using this guess. In some cases, the encoding cannot be determined. When this occurs, no guess is output. For conversion, you can force the input QV type with: -solexa input QV is solexa -illumina input QV is illumina -sanger input QV is sanger -o sanger-style-output.fastq If -stats is supplied, no QV analysis or conversion is performed, but some simple statistics are computed and output to stdout. canu-1.3/documentation/source/commands/fastqSample.rst000066400000000000000000000031111272335422300232660ustar00rootroot00000000000000fastqSample ~~~~~~ :: usage: fastqSample [opts] Input Specification -I NAME input name (prefix) of the reads -T T total number of mate pairs in the input (if not supplied, will be counted) -L L length of a single read (if not supplied, will be determined) -U reads are unmated, expected in *.u.fastq Output Specification -O NAME output name (prefix) of the reads (default is same as -I) -A automatically include coverage or number of reads in the output name -m L ignore reads shorter than L bases -max don't sample randomly, pick the longest reads Method 1: specify desired output coverage: -g G genome size -c C desired coverage in the output reads Method 2: specify desired number of output pairs -p N for mated reads, output 2N reads, or N pairs of reads for unmated reads, output N reads Method 3: specify a desired fraction of the input: -f F output F * T pairs of reads (T as above in -t option) 0.0 < F <= 1.0 Method 4: specify a desired total length -b B output reads/pairs until B bases is exceeded Samples reads from paired Illumina reads NAME.1.fastq and NAME.2.fastq and outputs: NAME.Cx.1.fastq and N.Cx.2.fastq (for coverage based sampling) NAME.n=N.1.fastq and N.n=N.2.fastq (for coverage based sampling) If -T is not supplied, the number of reads will be counted for you. ERROR: no name supplied with -I. ERROR: no method supplied with -c, -p, -f or -b canu-1.3/documentation/source/commands/fastqSimulate-sort.rst000066400000000000000000000003241272335422300246200ustar00rootroot00000000000000fastqSimulate-sort ~~~~~~ :: usage: fastqSimulate-sort -i1 in.1.fastq [-i2 in.2.fastq] -o1 out.1.fastq [-o2 out.2.fastq] ERROR: No in.1.fastq supplied with -i1. ERROR: No out.1.fastq supplied with -i1. canu-1.3/documentation/source/commands/fastqSimulate.rst000066400000000000000000000057641272335422300236500ustar00rootroot00000000000000fastqSimulate ~~~~~~ :: usage: fastqSimulate -f reference.fasta -o output-prefix -l read-length .... -f ref.fasta Use sequences in ref.fasta as the genome. -o name Create outputs name.1.fastq and name.2.fastq (and maybe others). -l len Create reads of length 'len' bases. -n n Create 'n' reads (for -se) or 'n' pairs of reads (for -pe and -mp). -x read-cov Set 'np' to create reads that sample the genome to 'read-cov' read coverage. -X clone-cov Set 'np' to create reads that sample the genome to 'clone-cov' clone coverage. -em err Reads will contain fraction mismatch error 'e' (0.01 == 1% error). -ei err Reads will contain fraction insertion error 'e' (0.01 == 1% error). -ed err Reads will contain fraction deletion error 'e' (0.01 == 1% error). -seed s Seed randomness with 32-bit integer s. -allowgaps Allow pairs to span N regions in the reference. By default, pairs are not allowed to span a gap. Reads are never allowed to cover N's. -allowns Allow reads to contain N regions. Implies -allowgaps -nojunction For -mp, do not create chimeric junction reads. Create only fully PE or fully MP reads. -normal p Output a normal-oriented (both forward or both reverse) pair with probability p. Only for -pe and -mp. -se Create single-end reads. -cc junkSize junkStdDev false Create chimeric single-end reads. The chimer is formed from two uniformly distributed positions in the reference. Some amount of random junk is inserted at the junction. With probability 'false' the read is not chimeric, but still the junk bases inserted in the middle. -pe shearSize shearStdDev Create paired-end reads, from fragments of size 'shearSize +- shearStdDev'. -mp insertSize insertStdDev shearSize shearStdDev enrichment Create mate-pair reads. The pairs will be 'insertSize +- insertStdDev' apart. The circularized insert is then sheared into fragments of size 'shearSize +- shearStdDev'. With probability 'enrichment' the fragment containing the junction is used to form the pair of reads. The junction location is uniformly distributed through this fragment. Reads are labeled as: tMP - a MP pair fMP - a PE pair aMP - a MP pair with junction in the first read bMP - a MP pair with junction in the second read cMP - a MP pair with junction in both reads (the reads overlap) Output QV's are the Sanger spec. ERROR: No fasta file (-f) supplied. ERROR: No output prefix (-o) supplied. ERROR: No type (-se or -pe or -mp) selected. canu-1.3/documentation/source/commands/filterCorrectionOverlaps.rst000066400000000000000000000021641272335422300260460ustar00rootroot00000000000000filterCorrectionOverlaps ~~~~~~ :: usage: filterCorrectionOverlaps [options] Rewrites an ovlStore, filtering overlaps that shouldn't be used for correcting reads. -G gkpStore input reads -O ovlStore input overlaps -S scoreFile output scores for each read, binary file, to 'scoreFile' per-read logging to 'scoreFile.log' (see -nolog) summary statistics to 'scoreFile.stats' (see -nostats) -c coverage retain at most this many overlaps per read -l length filter overlaps shorter than this length -e (min-)max filter overlaps outside this range of fraction error example: -e 0.20 filter overlaps above 20% error example: -e 0.05-0.20 filter overlaps below 5% error or above 20% error -nolog don't create 'scoreFile.log' -nostats don't create 'scoreFile.stats' ERROR: no gatekeeper store (-G) supplied. ERROR: no overlap store (-O) supplied. ERROR: no output scoreFile (-S) supplied. canu-1.3/documentation/source/commands/findErrors.rst000066400000000000000000000000301272335422300231200ustar00rootroot00000000000000findErrors ~~~~~~ :: canu-1.3/documentation/source/commands/gatekeeperCreate.rst000066400000000000000000000004001272335422300242440ustar00rootroot00000000000000gatekeeperCreate ~~~~~~ :: usage: gatekeeperCreate [...] -o gkpStore -o gkpStore create this gkpStore -minlength L discard reads shorter than L ERROR: no gkpStore (-o) supplied. ERROR: no input files supplied. canu-1.3/documentation/source/commands/gatekeeperDumpFASTQ.rst000066400000000000000000000022051272335422300245520ustar00rootroot00000000000000gatekeeperDumpFASTQ ~~~~~~ :: usage: gatekeeperDumpFASTQ [...] -o fastq-prefix -g gkpStore -G gkpStore -o fastq-prefix write files fastq-prefix.(libname).fastq, ... if fastq-prefix is '-', all sequences output to stdout if fastq-prefix ends in .gz, .bz2 or .xz, output is compressed -l libToDump output only read in library number libToDump (NOT IMPLEMENTED) -r id[-id] output only the single read 'id', or the specified range of ids -c clearFile clear range file from OBT modules -allreads if a clear range file, lower case mask the deleted reads -allbases if a clear range file, lower case mask the non-clear bases -onlydeleted if a clear range file, only output deleted reads (the entire read) -fastq output is FASTQ format (with extension .fastq, default) -fasta output is FASTA format (with extension .fasta) -nolibname don't include the library name in the output file name ERROR: no gkpStore (-G) supplied. ERROR: no output prefix (-o) supplied. canu-1.3/documentation/source/commands/gatekeeperDumpMetaData.rst000066400000000000000000000012121272335422300253510ustar00rootroot00000000000000gatekeeperDumpMetaData ~~~~~~ :: usage: gatekeeperDumpMetaData -G gkpStore [p] [...] -G gkpStore [p] dump reads from 'gkpStore', restricted to partition 'p', if supplied. -libs dump information about libraries -reads [-full] dump information about reads (-full also dumps some storage metadata) -stats dump summary statistics on reads -b id output starting at read/library 'id' -e id output stopping after read/library 'id' -r id output only the single read 'id' ERROR: no gkpStore (-G) supplied. canu-1.3/documentation/source/commands/gatekeeperPartition.rst000066400000000000000000000005161272335422300250220ustar00rootroot00000000000000gatekeeperPartition ~~~~~~ :: usage: gatekeeperPartition -G gkpStore -P partitionMapFile -G gkpStore path to gatekeeper store -P partFile file mapping read ID to partiton format: 'partition readID' ERROR: no gkpStore (-G) supplied. ERROR: no partition input (-P) supplied. canu-1.3/documentation/source/commands/generateCorrectionLayouts.rst000066400000000000000000000015011272335422300262120ustar00rootroot00000000000000generateCorrectionLayouts ~~~~~~ :: usage: generateCorrectionLayouts -G gkpStore -O ovlStore [ -T tigStore | -F ] ... -G gkpStore mandatory path to gkpStore -O ovlStore mandatory path to ovlStore -S file global score (binary) input file -T corStore output layouts to tigStore corStore -F output falconsense-style input directly to stdout -p name output prefix name, for logging and summary -b bgnID -e endID -rl file -L length minimum length of evidence overlaps -E erate maxerror rate of evidence overlaps -C coverage maximum coverage of evidence reads to emit -M length minimum length of a corrected read ERROR: no gkpStore input (-G) supplied. ERROR: no ovlStore input (-O) supplied. canu-1.3/documentation/source/commands/leaff.rst000066400000000000000000000044021272335422300220670ustar00rootroot00000000000000leaff ~~~~~~ :: usage: leaff [-f fasta-file] [options] SOURCE FILES -f file: use sequence in 'file' (-F is also allowed for historical reasons) -A file: read actions from 'file' SOURCE FILE EXAMINATION -d: print the number of sequences in the fasta -i name: print an index, labelling the source 'name' OUTPUT OPTIONS -6 <#>: insert a newline every 60 letters (if the next arg is a number, newlines are inserted every n letters, e.g., -6 80. Disable line breaks with -6 0, or just don't use -6!) -e beg end: Print only the bases from position 'beg' to position 'end' (space based, relative to the FORWARD sequence!) If beg == end, then the entire sequence is printed. It is an error to specify beg > end, or beg > len, or end > len. -ends n Print n bases from each end of the sequence. One input sequence generates two output sequences, with '_5' or '_3' appended to the ID. If 2n >= length of the sequence, the sequence itself is printed, no ends are extracted (they overlap). -C: complement the sequences -H: DON'T print the defline -h: Use the next word as the defline ("-H -H" will reset to the original defline -R: reverse the sequences -u: uppercase all bases SEQUENCE SELECTION -G n s l: print n randomly generated sequences, 0 < s <= length <= l -L s l: print all sequences such that s <= length < l -N l h: print all sequences such that l <= % N composition < h (NOTE 0.0 <= l < h < 100.0) (NOTE that you cannot print sequences with 100% N This is a useful bug). -q file: print sequences from the seqid list in 'file' -r num: print 'num' randomly picked sequences -s seqid: print the single sequence 'seqid' -S f l: print all the sequences from ID 'f' to 'l' (inclusive) -W: print all sequences (do the whole file) LONGER HELP -help analysis -help examples canu-1.3/documentation/source/commands/meryl.rst000066400000000000000000000140131272335422300221410ustar00rootroot00000000000000meryl ~~~~~~ :: usage: meryl [personality] [global options] [options] where personality is: -P -- compute parameters -B -- build table -S -- scan table -M -- "math" operations -D -- dump table -P: Given a sequence file (-s) or an upper limit on the number of mers in the file (-n), compute the table size (-t in build) to minimize the memory usage. -m # (size of a mer; required) -c # (homopolymer compression; optional) -p (enable positions) -s seq.fasta (seq.fasta is scanned to determine the number of mers) -n # (compute params assuming file with this many mers in it) Only one of -s, -n need to be specified. If both are given -s takes priority. -B: Given a sequence file (-s) and lots of parameters, compute the mer-count tables. By default, both strands are processed. -f (only build for the forward strand) -r (only build for the reverse strand) -C (use canonical mers, assumes both strands) -L # (DON'T save mers that occur less than # times) -U # (DON'T save mers that occur more than # times) -m # (size of a mer; required) -c # (homopolymer compression; optional) -p (enable positions) -s seq.fasta (sequence to build the table for) -o tblprefix (output table prefix) -v (entertain the user) By default, the computation is done as one large sequential process. Multi-threaded operation is possible, at additional memory expense, as is segmented operation, at additional I/O expense. Threaded operation: Split the counting in to n almost-equally sized pieces. This uses an extra h MB (from -P) per thread. -threads n (use n threads to build) Segmented, sequential operation: Split the counting into pieces that will fit into no more than m MB of memory, or into n equal sized pieces. Each piece is computed sequentially, and the results are merged at the end. Only one of -memory and -segments is needed. -memory mMB (use at most m MB of memory per segment) -segments n (use n segments) Segmented, batched operation: Same as sequential, except this allows each segment to be manually executed in parallel. Only one of -memory and -segments is needed. -memory mMB (use at most m MB of memory per segment) -segments n (use n segments) -configbatch (create the batches) -countbatch n (run batch number n) -mergebatch (merge the batches) Initialize the compute with -configbatch, which needs all the build options. Execute all -countbatch jobs, then -mergebatch to complete. meryl -configbatch -B [options] -o file meryl -countbatch 0 -o file meryl -countbatch 1 -o file ... meryl -countbatch N -o file meryl -mergebatch N -o file Batched mode can run on the grid. -sge jobname unique job name for this execution. Meryl will submit jobs with name mpjobname, ncjobname, nmjobname, for phases prepare, count and merge. -sgebuild "options" any additional options to sge, e.g., -sgemerge "options" "-p -153 -pe thread 2 -A merylaccount" N.B. - -N will be ignored N.B. - be sure to quote the options -M: Given a list of tables, perform a math, logical or threshold operation. Unless specified, all operations take any number of databases. Math operations are: min count is the minimum count for all databases. If the mer does NOT exist in all databases, the mer has a zero count, and is NOT in the output. minexist count is the minimum count for all databases that contain the mer max count is the maximum count for all databases add count is sum of the counts for all databases sub count is the first minus the second (binary only) abs count is the absolute value of the first minus the second (binary only) Logical operations are: and outputs mer iff it exists in all databases nand outputs mer iff it exists in at least one, but not all, databases or outputs mer iff it exists in at least one database xor outputs mer iff it exists in an odd number of databases Threshold operations are: lessthan x outputs mer iff it has count < x lessthanorequal x outputs mer iff it has count <= x greaterthan x outputs mer iff it has count > x greaterthanorequal x outputs mer iff it has count >= x equal x outputs mer iff it has count == x Threshold operations work on exactly one database. -s tblprefix (use tblprefix as a database) -o tblprefix (create this output) -v (entertain the user) NOTE: Multiple tables are specified with multiple -s switches; e.g.: meryl -M add -s 1 -s 2 -s 3 -s 4 -o all NOTE: It is NOT possible to specify more than one operation: meryl -M add -s 1 -s 2 -sub -s 3 will NOT work. -D: Dump the table (not all of these work). -Dd Dump a histogram of the distance between the same mers. -Dt Dump mers >= a threshold. Use -n to specify the threshold. -Dc Count the number of mers, distinct mers and unique mers. -Dh Dump (to stdout) a histogram of mer counts. -s Read the count table from here (leave off the .mcdat or .mcidx). canu-1.3/documentation/source/commands/mhapConvert.rst000066400000000000000000000006161272335422300233030ustar00rootroot00000000000000mhapConvert ~~~~~~ :: usage: mhapConvert [options] file.mhap[.gz] Converts mhap native output to ovb -o out.ovb output file -h id num base id and number of hash table reads (mhap output IDs 1 through 'num') -q id base id of query reads (mhap output IDs 'num+1' and higher) ERROR: no overlap files supplied canu-1.3/documentation/source/commands/ovStoreBucketizer.rst000066400000000000000000000024051272335422300245040ustar00rootroot00000000000000ovStoreBucketizer ~~~~~~ :: usage: ovStoreBucketizer -O asm.ovlStore -G asm.gkpStore -i file.ovb.gz -job j [opts] -O asm.ovlStore path to store to create -G asm.gkpStore path to gkpStore for this assembly -C config path to previously created ovStoreBuild config data file -i file.ovb.gz input overlaps -job j index of this overlap input file -F f use up to 'f' files for store creation -obt filter overlaps for OBT -dup filter overlaps for OBT/dedupe -e e filter overlaps above e fraction error -raw write uncompressed buckets DANGER DO NOT USE DO NOT USE DO NOT USE DANGER DANGER DANGER DANGER This command is difficult to run by hand. DANGER DANGER Use ovStoreCreate instead. DANGER DANGER DANGER DANGER DO NOT USE DO NOT USE DO NOT USE DANGER ERROR: No overlap store (-O) supplied. ERROR: No gatekeeper store (-G) supplied. ERROR: No input (-i) supplied. ERROR: No job index (-job) supplied. canu-1.3/documentation/source/commands/ovStoreBuild.rst000066400000000000000000000020661272335422300234370ustar00rootroot00000000000000ovStoreBuild ~~~~~~ :: usage: ovStoreBuild -O asm.ovlStore -G asm.gkpStore [opts] [-L fileList | *.ovb.gz] -O asm.ovlStore path to store to create -G asm.gkpStore path to gkpStore for this assembly -L fileList read input filenames from 'flieList' -F f use up to 'f' files for store creation -M g use up to 'g' gigabytes memory for sorting overlaps default 4; g-0.125 gb is available for sorting overlaps -e e filter overlaps above e fraction error -l l filter overlaps below l bases overlap length (needs gkpStore to get read lengths!) Non-building options: -evalues input files are evalue updates from overlap error adjustment -config out.dat don't build a store, just dump a binary partitioning file for ovStoreBucketizer ERROR: No overlap store (-o) supplied. ERROR: No gatekeeper store (-g) supplied. ERROR: No input overlap files (-L or last on the command line) supplied. canu-1.3/documentation/source/commands/ovStoreDump.rst000066400000000000000000000034101272335422300232770ustar00rootroot00000000000000ovStoreDump ~~~~~~ :: usage: ovStoreDump -G gkpStore -O ovlStore ... There are three modes of operation: -d [a[-b]] dump overlaps for reads a to b, inclusive -q a b report the a,b overlap, if it exists. -p a dump a picture of overlaps to fragment 'a'. FORMAT (for -d) -coords dump overlap showing coordinates in the reads (default) -hangs dump overlap showing dovetail hangs unaligned -raw dump overlap showing its raw native format (four hangs) -paf dump overlaps in miniasm/minimap format -binary dump overlap as raw binary data -counts dump the number of overlaps per read MODIFIERS (for -d and -p) -E erate Dump only overlaps <= erate fraction error. -L length Dump only overlaps that are larger than L bases (only for -p picture mode). -d5 Dump only overlaps off the 5' end of the A frag. -d3 Dump only overlaps off the 3' end of the A frag. -dC Dump only overlaps that are contained in the A frag (B contained in A). -dc Dump only overlaps that are containing the A frag (A contained in B). -v Report statistics (to stderr) on some dumps (-d). -unique Report only overlaps where A id is < B id, do not report both A to B and B to A overlap -best prefix Annotate picture with status from bogart outputs prefix.edges, prefix.singletons, prefix.edges.suspicious -noc With -best data, don't show overlaps to contained reads. -nos With -best data, don't show overlaps to suspicious reads. ERROR: no operation (-d, -q or -p) supplied. ERROR: no input gkpStore (-G) supplied. ERROR: no input ovlStore (-O) supplied. canu-1.3/documentation/source/commands/ovStoreIndexer.rst000066400000000000000000000020031272335422300237650ustar00rootroot00000000000000ovStoreIndexer ~~~~~~ :: usage: ovStoreIndexer ... -O x.ovlStore path to overlap store to build the final index for -F s number of slices used in bucketizing/sorting -t x.ovlStore explicitly test a previously constructed index -f when testing, also create a new 'idx.fixed' which might resolve rare problems -nodelete do not remove intermediate files when the index is successfully created DANGER DO NOT USE DO NOT USE DO NOT USE DANGER DANGER DANGER DANGER This command is difficult to run by hand. DANGER DANGER Use ovStoreCreate instead. DANGER DANGER DANGER DANGER DO NOT USE DO NOT USE DO NOT USE DANGER ERROR: No overlap store (-O) supplied. ERROR: One of -F (number of slices) or -t (test a store) must be supplied. canu-1.3/documentation/source/commands/ovStoreSorter.rst000066400000000000000000000021511272335422300236510ustar00rootroot00000000000000ovStoreSorter ~~~~~~ :: usage: ovStoreSorter ... -O x.ovlStore path to overlap store to build the final index for -G asm.gkpStore path to gkpStore for this assembly -F s number of slices used in bucketizing/sorting -job j m index of this overlap input file, and max number of files -M m maximum memory to use, in gigabytes -deleteearly remove intermediates as soon as possible (unsafe) -deletelate remove intermediates when outputs exist (safe) -force force a recompute, even if the output exists DANGER DO NOT USE DO NOT USE DO NOT USE DANGER DANGER DANGER DANGER This command is difficult to run by hand. DANGER DANGER Use ovStoreCreate instead. DANGER DANGER DANGER DANGER DO NOT USE DO NOT USE DO NOT USE DANGER ERROR: No overlap store (-O) supplied. ERROR: no slice number (-F) supplied. ERROR: no max job ID (-job) supplied. canu-1.3/documentation/source/commands/overlapConvert.rst000066400000000000000000000005501272335422300240230ustar00rootroot00000000000000overlapConvert ~~~~~~ :: usage: overlapConvert [options] file.ovb[.gz] -G gkpStore (needed for -coords, the default) -coords output coordiantes on reads -hangs output hangs on reads -raw output raw hangs on reads ERROR: -coords mode requires a gkpStore (-G) ERROR: no overlap files supplied canu-1.3/documentation/source/commands/overlapImport.rst000066400000000000000000000013061272335422300236550ustar00rootroot00000000000000overlapImport ~~~~~~ :: usage: overlapImport [options] ascii-ovl-file-input.[.gz] Required: -G name.gkpStore path to valid gatekeeper store Output options: -o file.ovb output file name -O name.ovlStore output overlap store Format options: -legacy 'CA8 overlapStore -d' format -coords 'overlapConvert -coords' format (not implemented) -hangs 'overlapConvert -hangs' format (not implemented) -raw 'overlapConvert -raw' format Input file can be stdin ('-') or a gz/bz2/xz compressed file. ERROR: need to supply a gkpStore (-G). ERROR: need to supply a format type (-legacy, -coords, -hangs, -raw). canu-1.3/documentation/source/commands/overlapInCore.rst000066400000000000000000000045201272335422300235630ustar00rootroot00000000000000overlapInCore ~~~~~~ :: * No kmer length supplied; -k needed! ERROR: No output file name specified USAGE: overlapInCore [options] -b in contig mode, specify the output file -c contig mode. Use 2 frag stores. First is for reads; second is for contigs -G do partial overlaps -h to specify fragments to put in hash table Implies LSF mode (no changes to frag store) -I designate a file of frag iids to limit olaps to (Contig mode only) -k if one or two digits, the length of a kmer, otherwise the filename containing a list of kmers to ignore in the hash table -l specify the maximum number of overlaps per fragment-end per batch of fragments. -m allow multiple overlaps per oriented fragment pair -M specify memory size. Valid values are '8GB', '4GB', '2GB', '1GB', '256MB'. (Not for Contig mode) -o specify output file name -P write protoIO output (if not -G) -r specify old fragments to overlap -t use parallel threads -u allow only 1 overlap per oriented fragment pair -w filter out overlaps with too many errors in a window -z skip the hopeless check --maxerate only output overlaps with fraction or less error (e.g., 0.06 == 6%) --minlength only output overlaps of or more bases --hashbits n Use n bits for the hash mask. --hashstrings n Load at most n strings into the hash table at one time. --hashdatalen n Load at most n bytes into the hash table at one time. --hashload f Load to at most 0.0 < f < 1.0 capacity (default 0.7). --maxreadlen n For batches with all short reads, pack bits differently to process more reads per batch. all reads must be shorter than n --hashstrings limited to 2^(30-m) Common values: maxreadlen 2048->hashstrings 524288 (default) maxreadlen 512->hashstrings 2097152 maxreadlen 128->hashstrings 8388608 --readsperbatch n Force batch size to n. --readsperthread n Force each thread to process n reads. canu-1.3/documentation/source/commands/overlapInCorePartition.rst000066400000000000000000000001351272335422300254530ustar00rootroot00000000000000overlapInCorePartition ~~~~~~ :: HASH: 0 reads or 0 length. REF: 0 reads or 0 length. canu-1.3/documentation/source/commands/overlapPair.rst000066400000000000000000000014711272335422300233010ustar00rootroot00000000000000overlapPair ~~~~~~ :: usage: overlapPair ... -G gkpStore Mandatory, path to gkpStore Inputs can come from either a store or a file. -O ovlStore -O ovlFile If from an ovlStore, the range of reads processed can be restricted. -b bgnID -e endID Outputs will be written to a store or file, depending on the input type -o ovlStore -o ovlFile -erate e Overlaps are computed at 'e' fraction error; must be larger than the original erate -partial Overlaps are 'overlapInCore -G' partial overlaps -memory m Use up to 'm' GB of memory -t n Use up to 'n' cores Advanced options: -invert Invert the overlap A <-> B before aligning (they are not re-inverted before output) canu-1.3/documentation/source/commands/prefixEditDistance-matchLimitGenerate.rst000066400000000000000000000003761272335422300303420ustar00rootroot00000000000000prefixEditDistance-matchLimitGenerate ~~~~~~ :: usage: prefixEditDistance-matchLimitGenerate minEvalue [maxEvalue [step]] computes overlapper probabilities for minEvalue <= eValue <= maxEvalue' eValue 100 == 0.01 fraction error == 1% error canu-1.3/documentation/source/commands/readConsensus.rst000066400000000000000000000014121272335422300236240ustar00rootroot00000000000000readConsensus ~~~~~~ :: usage: readConsensus ... -G gkpStore Mandatory, path to gkpStore Inputs can come from either an overlap or a tig store. -O ovlStore -T tigStore tigVers If from an ovlStore, the range of reads processed can be restricted. -b bgnID -e endID Outputs will be written as the full multialignment and the final consensus sequence -c output.cns -f output.fastq -erate e Overlaps are computed at 'e' fraction error; must be larger than the original erate -memory m Use up to 'm' GB of memory -t n Use up to 'n' cores ERROR: no gatekeeper (-G) supplied. ERROR: no inputs (-O or -T) supplied. ERROR: no outputs (-c or -f) supplied. canu-1.3/documentation/source/commands/simple.rst000066400000000000000000000001121272335422300222750ustar00rootroot00000000000000simple ~~~~~~ :: no input given with '-i' no output given with '-o' canu-1.3/documentation/source/commands/splitReads.rst000066400000000000000000000011241272335422300231220ustar00rootroot00000000000000splitReads ~~~~~~ :: usage: splitReads -G gkpStore -O ovlStore -Ci input.clearFile -Co output.clearFile -o outputPrefix] -G gkpStore path to read store -O ovlStore path to overlap store -o name output prefix, for logging -t bgn-end limit processing to only reads from bgn to end (inclusive) -Ci clearFile path to input clear ranges (NOT SUPPORTED) -Co clearFile path to ouput clear ranges -e erate ignore overlaps with more than 'erate' percent error -minlength l reads trimmed below this many bases are deleted canu-1.3/documentation/source/commands/tgStoreCoverageStat.rst000066400000000000000000000012611272335422300247510ustar00rootroot00000000000000tgStoreCoverageStat ~~~~~~ :: usage: tgStoreCoverageStat -G gkpStore -T tigStore version -o output-prefix [-s genomeSize] ... -G Mandatory, path G to a gkpStore directory. -T Mandatory, path T to a tigStore, and version V. -o Mandatory, prefix for output files. -s Optional, assume genome size S. -n Do not update the tigStore (default = do update). -u Do not estimate based on N50 (default = use N50). -L Be leniant; don't require reads start at position zero. No gatekeeper store (-G option) supplied. No input tigStore (-T option) supplied. No output prefix (-o option) supplied. canu-1.3/documentation/source/commands/tgStoreDump.rst000066400000000000000000000056741272335422300233030ustar00rootroot00000000000000tgStoreDump ~~~~~~ :: usage: tgStoreDump -G -T [opts] STORE SELECTION (mandatory) -G path to the gatekeeper store -T path to the tigStore, version, to use TIG SELECTION - if nothing specified, all tigs are reported - all ranges are inclusive. -tig A[-B] only dump tigs between ids A and B -unassembled only dump tigs that are 'unassembled' -bubbles only dump tigs that are 'bubbles' -contigs only dump tigs that are 'contigs' -nreads min max only dump tigs with between min and max reads -length min max only dump tigs with length between 'min' and 'max' bases -coverage c C g G only dump tigs with between fraction g and G at coverage between c and C example: -coverage 10 inf 0.5 1.0 would report tigs where half of the bases are at 10+ times coverage. DUMP TYPE - all dumps, except status, report on tigs selected as above -status the number of tigs in the store -tigs a list of tigs, and some information about them -consensus [opts] the consensus sequence, with options: -gapped report the gapped (multialignment) consensus sequence -fasta report sequences in FASTA format (the default) -fastq report sequences in FASTQ format -layout [opts] the layout of reads in each tig if '-o' is supplied, three files are created, otherwise just the layout is printed to stdout -gapped report the gapped (multialignment) positions -o outputPrefix write plots to 'outputPrefix.*' in the current directory -multialign [opts] the full multialignment, output is to stdout -w width width of the page -s spacing spacing between reads on the same line -sizes [opts] size statistics -s genomesize denominator to use for n50 computation -coverage [opts] read coverage plots, one plot per tig -o outputPrefix write plots to 'outputPrefix.*' in the current directory -depth [opts] a histogram of depths -single one histogram per tig -overlap read overlaps -thin overlap report regions where the (thickest) read overlap is less than 'overlap' bases -overlaphistogram a histogram of the thickest overlaps used -o outputPrefix write plots to 'outputPrefix.*' in the current directory canu-1.3/documentation/source/commands/tgStoreFilter.rst000066400000000000000000000000731272335422300236070ustar00rootroot00000000000000tgStoreFilter ~~~~~~ :: this is obsolete. do not use. canu-1.3/documentation/source/commands/tgStoreLoad.rst000066400000000000000000000021631272335422300232430ustar00rootroot00000000000000tgStoreLoad ~~~~~~ :: usage: tgStoreLoad -G -T [input.cns] -G Path to the gatekeeper store -T Path to the tigStore and version to add tigs to -L Load the tig(s) from files listed in 'file-of-files' -n Don't replace, just report what would have happened The primary operation is to replace tigs in the store with ones in a set of input files. The input files can be either supplied directly on the command line or listed in a text file (-L). A new store is created if one doesn't exist, otherwise, whatever tigs are there are replaced with those in the -R file. If version 'v' doesn't exist, it is created. Even if -n is supplied, a new store is created if one doesn't exist. To add a new tig, give it a tig id of -1. New tigs must be added to the latest version. To delete a tig, remove all children, and set the number of them to zero. ERROR: no gatekeeper store (-G) supplied. ERROR: no tig store (-T) supplied. ERROR: no input tigs (-R) supplied. canu-1.3/documentation/source/commands/tgTigDisplay.rst000066400000000000000000000001061272335422300234130ustar00rootroot00000000000000tgTigDisplay ~~~~~~ :: usage: tgTigDisplay -G gkpStore -t tigFile canu-1.3/documentation/source/commands/trimReads.rst000066400000000000000000000015011272335422300227410ustar00rootroot00000000000000trimReads ~~~~~~ :: usage: trimReads -G gkpStore -O ovlStore -Co output.clearFile -o outputPrefix -G gkpStore path to read store -O ovlStore path to overlap store -o name output prefix, for logging -t bgn-end limit processing to only reads from bgn to end (inclusive) -Ci clearFile path to input clear ranges (NOT SUPPORTED) -Co clearFile path to ouput clear ranges -e erate ignore overlaps with more than 'erate' percent error -ol l the minimum evidence overlap length -oc c the minimum evidence overlap coverage evidence overlaps must overlap by 'l' bases to be joined, and must be at least 'c' deep to be retained -minlength l reads trimmed below this many bases are deleted canu-1.3/documentation/source/commands/updateDocs.sh000066400000000000000000000004671272335422300227160ustar00rootroot00000000000000for file in `ls *.rst`; do cmd=`echo $file |sed s/.rst//g` export PATH=$PATH:../../../Darwin-amd64/bin OUTPUT=`$cmd 2>&1 |awk '{print " "$0}'` echo "$cmd" > $file echo "~~~~~~" >> $file echo "" >> $file echo "::" >> $file echo "" >> $file echo "$OUTPUT" >> $file done canu-1.3/documentation/source/commands/utgcns.rst000066400000000000000000000066141272335422300223240ustar00rootroot00000000000000utgcns ~~~~~~ :: usage: utgcns [opts] INPUT -G g Load reads from gkStore 'g' -T t v p Load unitigs from tgStore 't', version 'v', partition 'p'. Expects reads will be in gkStore partition 'p' as well Use p='.' to specify no partition -t file Test the computation of the unitig layout in 'file' 'file' can be from: 'tgStoreDump -d layout' (human readable layout format) 'utgcns -L' (human readable layout format) 'utgcns -O' (binary multialignment format) -p package Load unitig and read from 'package' created with -P. This is usually used by developers. ALGORITHM -quick No alignments, just paste read sequence into the unitig positions. This is very fast, but the consensus sequence is formed from a mosaic of read sequences, and there can be large indel. This is useful for checking intermediate assembly structure by mapping to reference, or possibly for use as input to a polishing step. -pbdagcon Use pbdagcon (https://github.com/PacificBiosciences/pbdagcon). This is fast and robust. It is the default algorithm. It does not generate a final multialignment output (the -v option will not show anything useful). -utgcns Use utgcns (the original Celera Assembler consensus algorithm) This isn't as fast, isn't as robust, but does generate a final multialign output. OUTPUT -O results Write computed tigs to binary output file 'results' -L layouts Write computed tigs to layout output file 'layouts' -A fasta Write computed tigs to fasta output file 'fasta' -Q fastq Write computed tigs to fastq output file 'fastq' -P package Create a copy of the inputs needed to compute the unitigs. This file can then be sent to the developers for debugging. The unitig(s) are not processed and no other outputs are created. Ideally, only one unitig is selected (-u, below). TIG SELECTION (if -T input is used) -u b Compute only unitig ID 'b' (must be in the correct partition!) -u b-e Compute only unitigs from ID 'b' to ID 'e' -f Recompute unitigs that already have a multialignment -maxlength l Do not compute consensus for unitigs longer than l bases. PARAMETERS -e e Expect alignments at up to fraction e error -em m Don't ever allow alignments more than fraction m error -l l Expect alignments of at least l bases -maxcoverage c Use non-contained reads and the longest contained reads, up to C coverage, for consensus generation. The default is 0, and will use all reads. LOGGING -v Show multialigns. -V Enable debugging option 'verbosemultialign'. ERROR: No gkpStore (-G) and no package (-p) supplied. ERROR: No tigStore (-T) OR no test unitig (-t) OR no package (-p) supplied. canu-1.3/documentation/source/conf.py000066400000000000000000000210251272335422300177560ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # canu documentation build configuration file, created by # sphinx-quickstart on Wed Aug 26 18:41:08 2015. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import sys import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.todo', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'canu' copyright = u'2015, Adam Phillippy, Sergey Koren, Brian Walenz' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '1.3' # The full version, including alpha/beta/rc tags. release = '1.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # Build using the RTD theme, if not on RTD. # https://read-the-docs.readthedocs.org/en/latest/theme.html # https://github.com/snide/sphinx_rtd_theme # on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme html_theme = 'sphinx_rtd_theme' html_theme_path = [ "/usr/local/lib/python2.7/site-packages", ] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'canudoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'canu.tex', u'canu Documentation', u'Adam Phillippy, Sergey Koren, Brian Walenz', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'canu', u'canu Documentation', [u'Adam Phillippy, Sergey Koren, Brian Walenz'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'canu', u'canu Documentation', u'Adam Phillippy, Sergey Koren, Brian Walenz', 'canu', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False canu-1.3/documentation/source/faq.rst000066400000000000000000000135161272335422300177660ustar00rootroot00000000000000 .. _faq: Canu FAQ ======================== **Q**: What resources does Canu require for a bacterial genome assembly? A mammalian assembly? **A**: Canu is designed to scale resources to the system it runs on. It will report if the a system does not meet the minimum requirements for a given genome size. Typically, a bacterial genome can be assembled in 1-10 cpu hours, depending on coverage (~20 min on 16-cores) and 4GB of ram (8GB is recommended). A mammalian genome (such as human) can be assembled in 10-25K cpu hours, depending on coverage (a grid environment is recommended) and at least one machine with 64GB of ram (128GB is recommended). **Q**: What parameters should I use for my genome? Sequencing type? **A**: By default, Canu is designed to be universal on a large range of PacBio (C2-P6-C4) and Oxford Nanopore (R6-R9) data. You can adjust parameters to increase efficiency for your datatype. For example, for higher coverage PacBio datasets, especially from inbred samples, you can decrease the error rate (``errorRate=0.013``). For recent Nanopore data (R9) 2D data, you can also decrease the default error rate (``errorRate=0.013``). With R7 1D sequencing data, multiple rounds of error correction are helpful. This should not be necessary for sequences over 85% identity. You can run just the correction from Canu with the options :: -correct corOutCoverage=500 corMinCoverage=0 corMhapSensitivity=high for 5-10 rounds, supplying the asm.correctedReads.fasta.gz output from round ``i-1`` to round ``i``. Assemble with :: -nanopore-corrected errorRate=0.1 utgGraphDeviation=50 **Q**: How do I run Canu on my SLURM/SGE/PBS/LSF/Torque system? **A**: Canu will auto-detect and configure itself to submit on most grids. If your grid requires special options (such as a partition on SLURM or an account code on SGE, specify it with ``gridOptions=""`` which will passed to the sheduler by Canu. If you have a grid system but prefer to run locally, specify useGrid=false **Q**: My asm.contigs.fasta is empty, why? **A**: By default, canu will split the final output into three files: asm.contigs.fasta Everything which could be assembled and is part of the primary assembly, including both unique and repetitive elements. Each contig has several flags included on the fasta def line:: asm.bubbles.fasta alternate paths in the graph which could not be merged into the primary assembly. asm.unassembled.fasta reads/tigs which could not be incorporated into the primary or bubble assemblies. It is possible for tigs comprised of multiple reads to end up in asm.unassembled.fasta. The default filtering eliminates anything with < 2 reads, shorter than 1000bp, or comprised of mostly a single sequence (>75%). The filtering is controlled by the contigFilter parameter which takes 5 values. :: contigFilter minReads minLength singleReadSpan lowCovSpan lowCovDepth The default filtering is ``2 1000 0.75 0.75 2``. If you are assembling amplified data or viral data, it is possible your assembly will be flagged as unassembled. In those cases, you can turn off the filtering with the parameters :: contigFilter="2 1000 1.0 1.0 2" **Q**: Why is my assembly is missing my favorite short plasmid X? **A**: The first step in Canu is to find high-error overlaps and generate corrected sequences for subsequent assembly. This is currently the fastest step in Canu. By default, only the longest 40X of data (based on the specified genome size) is used for correction. If you have a dataset with uneven coverage or small plasmids, correcting the longest 40X may not give you sufficient coverage of your genome/plasmid. In these cases, you can set :: corOutCoverage=1000 Or any large value greater than your total input coverage which will correct and assemble all input data, at the expense of runtime. This option is also recommended for metagenomic datasets where all data is useful for assembly. **Q**: Why do I get only 30X of corrected data? **A**: By default, only the longest 40X of data (based on the specified genome size) is used for correction. Typically, some reads are trimmed during correction due to being chimeric or having erroneous sequence, resulting in a loss of 20-25% (30X output). You can force correction to be non-lossy by setting :: corMinCoverage=0 In which case the corrected reads output will be the same length as the input data, keeping any high-error unsupported bases. Canu will trim these in downstream steps before assembly. **Q**: What is the minimum coverage required to run Canu? **A**: We have found that on eukaryotic genomes >=20X typically begins to outperform current hybrid methods. For low coverage datasets (<=30X) we recommend the following parameters :: corMinCoverage=0 errorRate=0.035 For high-coverage datasets (typically >=60X) you can decrease the error rate since the higher number of reads should allow sufficient assembly from only the best subset :: errorRate=0.013 However, the above is mainly an optimization for speed and will not affect your assembly continuity. **Q**: My genome is AT/GC rich, do I need to adjust parameters? **A**: On bacterial genomes, typically no. On repetitive genomes with AT<=25 or 75>=AT (or GC) the sequence biases the Jaccard estimate used by MHAP. In those cases setting :: corMaxEvidenceErate=0.15 has been sufficient to correct for the bias in our testing. In general, with high coverage repetitive genomes (such as plants) it can be beneficial to set the above parameter as it will eliminate repetitive matches, speed up the assembly, and sometime improve unitigs. canu-1.3/documentation/source/history.rst000066400000000000000000000067231272335422300207220ustar00rootroot00000000000000.. _history: Software Background ==================== Canu is derived from the Celera Assembler. The Celera assembler [Myers 2000] was designed to reconstruct mammalian chromosomal DNA sequences from the short fragments of a whole genome shotgun sequencing project. The Celera Assembler was used to produce reconstructions of several large genomes, namely those of Homo sapiens [Venter 2001], Mus musculus [Mural 2002], Rattus norvegicus [unpublished data], Canis familiaris [Kirkness 2003], Drosophila melanogaster [Adams 2000], and Anopheles gambiae [Holt 2001]. The Celera Assembler was shown to be very accurate when its reconstruction of the human genome was compared to independent reconstructions completed later [Istrail 2004]. It was used to reconstructing one of the first large-scale metagenomic projects [Venter 2004, Rusch 2007] and a diploid human reference [Levy 2007, Denisov 2008]. It was adapted to 454 Pyrosequencing [Miller 2008] and PacBio sequencing [Koren 2011], demonstrating finished bacterial genomes [Koren 2013] and efficient algorithms for eukaryotic assembly [Berlin 2015]. In 2015 Canu was forked from Celera Assembler and specialized for single-molecule high-noise sequences. The Celera Assembler codebase is no longer maintained. Canu is a pipeline consisting of several executable programs and perl driver scripts. The source code includes programs in the C++ language with Unix make scripts. The original Celera Assembler was designed to run under Compaq(R) Tru64(R) Unix with access to 32GB RAM. It has also run under IBM(R) AIX(R) and Red Hat Linux. The Celera Assembler was released under the GNU General Public License, version 2 as as supplement to the publication [Istrail 2004]. For the most recent license information please see README.licences References -------------------- - Adams et al. (2000) The Genome Sequence of Drosophila melanogaster. Science 287 2185-2195. - Myers et al. (2000) A Whole-Genome Assembly of Drosophila. Science 287 2196-2204. - Venter et al. (2001) The Sequence of the Human Genome. Science 291 1304-1351. - Mural et al. (2002) A Comparison of Whole-Genome Shotgun-Derived Mouse Chromosome 16 and the Human Genome. Science 296 1661-1671. - Holt et al. (2002) The Genome Sequence of the Malaria Mosquito Anophelesd gambiae. Science 298 129-149. - Istrail et al. (2004) Whole Genome Shotgun Assembly and Comparison of Human Genome Assemblies. PNAS 101 1916-1921. - Kirkness et al. (2003) The Dog Genome: Survey Sequencing and Comparative Analysis. Science 301 1898-1903. - Venter et al. (2004) Environmental genome shotgun sequencing of the Sargasso Sea. Science 304 66-74. - Levy et al. (2007) The Diploid Genome Sequence of an Individual Human. PLoS Biology 0050254 - Rusch et al. (2007) The Sorcerer II Global Ocean Sampling Expedition: Northwest Atlantic through Eastern Tropical Pacific. PLoS Biology 1821060. - Denisov et al. (2008) Consensus Generation and Variant Detection by Celera Assembler. Bioinformatics 24(8):1035-40 - Miller et al. (2008) Aggressive Assembly of Pyrosequencing Reads with Mates. Bioinformatics 24(24):2818-2824 - Koren et al. (2012) Hybrid error correction and de novo assembly of single-molecule sequencing reads, Nature Biotechnology, July 2012. - Koren et al. (2013) Reducing assembly complexity of microbial genomes with single-molecule sequencing, Genome Biology 14:R101. - Berlin et. al. (2015) Assembling Large Genomes with Single-Molecule Sequencing and Locality Sensitive Hashing. Nature Biotechnology. (2015). canu-1.3/documentation/source/index.rst000066400000000000000000000032141272335422300203200ustar00rootroot00000000000000 .. introduction what this is quick start one PacBio SMRT cell from ecoli pipeline (not reference, designed to be read through) overview introduce modular pipeline introduce gkpStore, overlaps, ovlStore, tigStore read correction read trimming unitig construction local vs grid mode canu option reference each option, in detail, grouped by function canu executable reference each binary, in detail, alphabetical option index (alphabetical) history Canu ==== .. toctree:: :hidden: quick-start faq tutorial pipeline parameter-reference command-reference history `Canu `_ is a fork of the Celera Assembler designed for high-noise single-molecule sequencing (such as the PacBio RSII or Oxford Nanopore MinION). You can `download `_ a release. If you encounter any issues, please report them using the `github issues `_ page. * :ref:`Quick Start ` - no experience or data required, download and assemble *Escherichia coli* today! * :ref:`FAQ ` - Frequently asked questions * :ref:`Canu tutorial ` - a gentle introduction to the complexities of canu. * :ref:`Canu pipeline ` - what, exactly, is canu doing, anyway? * :ref:`Canu Parameter Reference ` - all the paramters, grouped by function. * :ref:`Canu Command Reference ` - all the commands that canu runs for you. * :ref:`Canu History ` - the history of the Canu project. canu-1.3/documentation/source/parameter-reference.rst000066400000000000000000000504161272335422300231330ustar00rootroot00000000000000 .. _parameter-reference: Canu Parameter Reference ======================== To get the most up-to-date options, run canu -options The default values below will vary based on the input data type and genome size. .. _genomeSize: Global Options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The catch all category. errorRate The expected error rate, as fraction error, in the corrected reads, set by default based on data type, typically not changed by the user. genomeSize An estimate of the size of the genome. Common suffices are allowed, for example, 3.7m or 2.8g. Required. canuIteration Which parallel iteration is being attempted. canuIterationMax How many parallel iterations to try. Ideally, the parallel jobs, run under grid control, would all finish successfully on the first try. Sometimes, jobs fail due to other jobs exhausting resources (memory), or by the node itself failing. In this case, canu will launch the jobs again. This parameter controls how many times it tries. Process Control ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ showNext Report the first major command that would be run, but don't run it. Processing to get to that command, for example, checking the output of the previous command or preparing inputs for the next command, is still performed. stopBefore Stop processing just before this stage would execute. The stage is configured, and the command logged to the standard output, before stopping. For grid-based stages, e.g., overlapper, the grid submit command is reported. If the stage has finished successfully, it will not stop. Only one stage may be supplied to stopBefore. Valid stages to stopBefore are: - gatekeeper - meryl - overlap - correction - overlapErrorAdjustment - trimReads - splitReads - unitig - consensusConfigure - consensus - output The default value is 'undef'. stopAfter Stop processing after this stage completes. The stage will typically stop BEFORE a summary of its processing is reported in the canu chatter. Valid stages top stopAfter are: - gatekeeper - meryl - overlapConfigure - overlap - correction - unitig - consensusConfigure - consensus - consensusLoad - consensusFilter General Options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ pathMap A text file containing lines that map a hostname to a path to the assembler binaries. This can be used to provide fine-grained binary directories, for example, when incompatible versions of the operating system are used, or when canu is installed in a non-standard way. The hostname must be the same as returned by 'uname -n'. For example:: grid01 /grid/software/canu/Linux-amd64/bin/ devel01 /devel/canu/Linux-amd64/bin/ shell A path to a Bourne shell, to be used for executing scripts. By default, '/bin/sh', which is typically the same as 'bash'. C shells (csh, tcsh) are not supported. java A path to a Java application launcher of at least version 1.8. Cleanup Options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ saveOverlaps If set, do not remove raw overlap output from either mhap or overlapInCore. Normally, this output is removed once the overlaps are loaded into an overlap store. saveReadCorrections If set, do not remove intermediate outputs. Normally, intermediate files are removed once they are no longer needed. NOT IMPLEMENTED. saveMerCounts If set, do not remove meryl binary databases. Overlapper Configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Overlaps are generated for three purposes: read correction, read trimming and unitig construction. The algorithm and parameters used can be set independently for each set of overlaps. Two overlap algorithms are in use. One, mhap, is typically applied to raw uncorrected reads and returns alignment-free overlaps with imprecise extents. The other, the original overlapper algorithm 'ovl', returns alignments but is much more expensive. There are three sets of parameters, one for the 'mhap' algorithm, one for the 'ovl' algorithm, and one for the 'minimap' algorithm. Parameters used for a specific type of overlap are set by a prefix on the option: 'cor' for read correction, 'obt' for read trimming ('overlap based trimming') or 'utg' for unitig construction. For example, 'corOverlapper=ovl' would set the overlapper used for read correction to the 'ovl' algorithm. {prefix}Overlapper Specify which overlap algorith, 'mhap' or 'ovl' or 'minimap'. The default is to use 'mhap' for 'cor' and 'ovl' for both 'obt' and 'utg'. Overlapper Configuration, ovl Algorithm --------------------------------------- {prefix}OvlErrorRate Overlaps above this error rate are not computed. {prefix}OvlFrequentMers Do not seed overlaps with these kmers (fasta format). {prefix}OvlHashBits Width of the kmer hash. Width 22=1gb, 23=2gb, 24=4gb, 25=8gb. Plus 10b per corOvlHashBlockLength. {prefix}OvlHashBlockLength Amount of sequence (bp to load into the overlap hash table. {prefix}OvlHashLoad Maximum hash table load. If set too high, table lookups are inefficent; if too low, search overhead dominates run time. {prefix}OvlMerDistinct K-mer frequency threshold; the least frequent fraction of distinct mers can seed overlaps. {prefix}OvlMerSize K-mer size for seeds in overlaps. {prefix}OvlMerThreshold K-mer frequency threshold; mers more frequent than this count are not used to seed overlaps. {prefix}OvlMerTotal K-mer frequency threshold; the least frequent fraction of all mers can seed overlaps. {prefix}OvlRefBlockLength Amount of sequence (bp to search against the hash table per batch. {prefix}OvlRefBlockSize Number of reads to search against the hash table per batch. Overlapper Configuration, mhap Algorithm ---------------------------------------- {prefix}MhapBlockSize Number of reads per 1GB block. Memory * size is loaded into memory per job. {prefix}MhapMerSize K-mer size for seeds in mhap. {prefix}ReAlign Compute actual alignments from mhap overlaps; 'raw' from mhap output; uses either obtErrorRate or ovlErrorRate, depending on which overlaps are computed) {prefix}MhapSensitivity Coarse sensitivity level: 'normal' or 'high' or 'fast'. Overlapper Configuration, mhap Algorithm ---------------------------------------- {prefix}MMapBlockSize Number of reads per 1GB block. Memory * size is loaded into memory per job. {prefix}MMapMerSize K-mer size for seeds in minimap. {prefix}ReAlign Compute actual alignments from minimap overlaps; 'raw' from mhap output; uses either obtErrorRate or ovlErrorRate, depending on which overlaps are computed) Overlap Store ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The overlap algorithms return overlaps in an arbitrary order. The correction, trimming and assembly algorithms usually need to know all overlaps for a single read. The overlap store duplicates each overlap, sorts them by the first ID, and stores them for quick retrieval of all overlaps for a single read. ovsMemory How much memory, in gigabytes, to use for constructing overlap stores. ovsMethod Two construction algorithms are supported. One uses a single data stream, and is faster for small and moderate size assemblies. The other uses parallel data streams and can be faster (depending on your network disk bandwitdh) for moderate and large assemblies. Meryl ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The 'meryl' algorithm counts the occurrences of kmers in the input reads. It outputs a FASTA format list of frequent kmers, and (optionally) a binary database of the counts for each kmer in the input. Meryl can run in (almost) any memory size, by splitting the computation into smaller (or larger) chunks. merylMemory Amount of memory, in gigabytes, to use for counting kmers. merylThreads Number of compute threads to use for kmer counting. Overlap Based Trimming ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ obtErrorRate Stringency of overlaps to use for trimming trimReadsOverlap Minimum overlap between evidence to make contiguous trim. trimReadsCoverage Minimum depth of evidence to retain bases. .. _grid-engine: Grid Engine Support ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Canu supports Sun/Open/Univa Grid Engine (SGE), Portable Batch System (PBS), Platform Computing's Load Sharing Facility (LSF), and the Simple Linux Utility for Resource Management (SLURM). Most of the compute intensive stages can run under grid control. The useGrid* options control which algorithms run in parallel on the grid. useGrid Master control. If 'false', no algorithms will run under grid control. Does not change the value of the other useGrid options. If 'remote', jobs are configured for grid execution, but not submitted. A message, with commands to launch the job, is reported and canu halts execution. Note that the host used to run canu for 'remote' execution must know about the grid, that is, it must be able to submit jobs to the grid. There are many options for configuring a new grid ('gridEngine*') and for configuring how canu configures its computes to run under grid control ('gridOptions*'). The grid engine to use is specified with the 'gridEngine' option. gridEngine Which grid engine to use. Auto-detected. Possible choices are 'sge', 'pbs', 'lsf' or 'slurm'. NOTE: 'lsf' support is untested. .. _grid-engine-config: Grid Engine Configuration ------------------------- There are many options to configure support for a new grid engine, and we don't describe them fully. If you feel the need to add support for a new engine, please contact us. That said, file ``src/pipeline/canu/Defaults.pm`` lists a whole slew of parameters that are used to build up grid commands, they all start with ``gridEngine``. For each grid, these parameters are defined in the various ``src/pipeline/Grid_*.pm`` modules. The parameters are used in ``src/pipeline/canu/Execution.pm``. For SGE grids, two options are sometimes necessary to tell canu about pecularities of your grid: ``gridEngineThreadsOption`` describes how to request multiple cores, and ``gridEngineMemoryOption`` describes how to request memory. Usually, canu can figure out how to do this, but sometimes it reports an error such as:: -- WARNING: Couldn't determine the SGE parallel environment to run multi-threaded codes. -- Valid choices are (pick one and supply it to canu): -- gridEngineThreadsOption="-pe make THREADS" -- gridEngineThreadsOption="-pe make-dedicated THREADS" -- gridEngineThreadsOption="-pe mpich-rr THREADS" -- gridEngineThreadsOption="-pe openmpi-fill THREADS" -- gridEngineThreadsOption="-pe smp THREADS" -- gridEngineThreadsOption="-pe thread THREADS" or:: -- WARNING: Couldn't determine the SGE resource to request memory. -- Valid choices are (pick one and supply it to canu): -- gridEngineMemoryOption="-l h_vmem=MEMORY" -- gridEngineMemoryOption="-l mem_free=MEMORY" If you get such a message, just add the appropriate line to your canu command line. Both options will replace the uppercase text (THREADS or MEMORY) with the value canu wants when the job is submitted. For ``gridEngineMemoryOption``, any number of ``-l`` options can be supplied; we could use ``gridEngineMemoryOption="-l h_vmem=MEMORY -l mem_free=MEMORY"`` to request both ``h_vmem`` and ``mem_free`` memory. .. _grid-options: Grid Options ------------ To run on the grid, each stage needs to be configured - to tell the grid how many cores it will use and how much memory it needs. Some support for this is automagic (for example, overlapInCore and mhap know how to do this), others need to be manually configured. Yes, it's a problem, and yes, we want to fix it. The gridOptions* parameters supply grid-specific opitons to the grid submission command. gridOptions Grid submission command options applied to all grid jobs gridOptionsJobName Grid submission command jobs name suffix gridOptionsCNS Grid submission command options applied to unitig consensus jobs gridOptionsCOR Grid submission command options applied to read correction jobs gridOptionsExecutive Grid submission command options applied to master script jobs gridOptionsOEA Grid submission command options applied to overlap error adjustment jobs gridOptionsRED Grid submission command options applied to read error detection jobs gridOptionsOVB Grid submission command options applied to overlap store bucketizing jobs gridOptionsOVS Grid submission command options applied to overlap store sorting jobs gridOptionsCORMHAP Grid submission command options applied to mhap overlaps for correction jobs gridOptionsCOROVL Grid submission command options applied to overlaps for correction jobs gridOptionsOBTMHAP Grid submission command options applied to mhap overlaps for trimming jobs gridOptionsOBTOVL Grid submission command options applied to overlaps for trimming jobs gridOptionsUTGMHAP Grid submission command options applied to mhap overlaps for unitig construction jobs gridOptionsUTGOVL Grid submission command options applied to overlaps for unitig construction jobs Algorithm Selection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Several algorithmic components of canu can be disabled, based on the type of the reads being assebmled, the type of processing desired, or the amount of comput resources available. Overlap enableOEA Do overlap error adjustment - comprises two steps: read error detection (RED and overlap error adjustment (OEA WHERE IS OBT?? Algorithm Execution Method -------------------------- Each of the high compute stages can be computed either on a grid or in parallel on the local machine. Most algorithms will respect a given maximum memory usage. Most algorithms can support more than a single thread of computation. When the grid engine is not used, more than one task can be run at a time. BUG: Local execution doesn't pay attention to memory option. For execution locally, three parameters describe the task: {prefix}Concurrency Set the number of tasks that can run at the same time, when running without grid support. Available prefixes are: - master - cns - cor - cormhap - obtmhap - utgmhap - corovl - obtovl - utgovl - cormmap - obtmmap - utgmmap - oea - ovb - ovs - red {prefix}Threads Set the number of compute threads used per task. Available prefixes are: - master - bat - cns - cor - cormhap - obtmhap - utgmhap - corovl - obtovl - utgovl - cormmap - obtmmap - utgmmap - ovb - ovs - red - oea {prefix}Memory Set the amount of memory, in GB, to use for each job in a task. Available prefixes are: - master - bat - ovb - ovs - cns - cor - cormhap - obtmhap - utgmhap - corovl - obtovl - utgovl - cormmap - obtmmap - utgmmap - red - oea Overlap Error Adjustment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ red = Read Error Detection oea = Overlap Error Adjustment oeaBatchLength Number of bases per overlap error correction batch oeaBatchSize Number of reads per overlap error correction batch redBatchLength Number of bases per fragment error detection batch redBatchSize Number of reads per fragment error detection batch Unitigger ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ unitigger Which unitig construction algorithm to use. Only "bogart" is supported. batOptions Advanced options to bogart Consensus Partitioning ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ STILL DONE BY UNITIGGER, NEED TO MOVE OUTSIDE cnsConsensus Which algorithm to use for computing consensus sequences. Only 'utgcns' is supported. cnsPartitions Compute conseus by splitting the tigs into N partitions. cnsPartitionMin Don't make a paritition with fewer than N reads cnsMaxCoverage Limit unitig consensus to at most this coverage. cnsErrorRate Possibly unused. Read Correction ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The first step in Canu is to find high-error overlaps and generate corrected sequences for subsequent assembly. This is currently the fastest step in Canu. By default, only the longest 40X of data (based on the specified genome size) is used for correction. Typically, some reads are trimmed during correction due to being chimeric or having erroneous sequence, resulting in a loss of 20-25% (30X output). You can force correction to be non-lossy by setting :: corMinCoverage=0 In which case the corrected reads output will be the same length as the input data, keeping any high-error unsupported bases. Canu will trim these in downstream steps before assembly. If you have a dataset with uneven coverage or small plasmids, correcting the longest 40X may not give you sufficient coverage of your genome/plasmid. In these cases, you can set :: corOutCoverage=400 Or any large value greater than your total input coverage which will correct and assemble all input data, at the expense of runtime. corConsensus Which algorithm to use for computing read consensus sequences. Only 'falcon' and 'falconpipe' are supported. corPartitions Partition read correction into N jobs corPartitionMin Don't make a read correction partition with fewer than N reads corMinEvidenceLength Limit read correction to only overlaps longer than this; default: unlimited corMinCoverage Limit read correction to regions with at least this minimum coverage. Split reads when coverage drops below threshold. corMaxEvidenceErate Limit read correction to only overlaps at or below this fraction error; default: unlimited corMaxEvidenceCoverageGlobal Limit reads used for correction to supporting at most this coverage; default: 1.0 * estimated coverage corMaxEvidenceCoverageLocal Limit reads being corrected to at most this much evidence coverage; default: 10 * estimated coverage corOutCoverage Only correct the longest reads up to this coverage; default 40 corFilter Method to filter short reads from correction; 'quick' or 'expensive' or 'none' falconSense Path to fc_consensus.py or falcon_sense.bin Output Filtering ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, canu will split the final output into three files: asm.contigs.fasta Everything which could be assembled and is part of the primary assembly, including both unique and repetitive elements. Each contig has several flags included on the fasta def line:: asm.bubbles.fasta alternate paths in the graph which could not be merged into the primary assembly. asm.unassembled.fasta reads/tigs which could not be incorporated into the primary or bubble assemblies. It is possible for tigs comprised of multiple reads to end up in asm.unassembled.fasta. The default filtering eliminates anything with < 2 reads, shorter than 1000bp, or comprised of mostly a single sequence (>75%). The filtering is controlled by the contigFilter parameter which takes 5 values. :: contigFilter minReads minLength singleReadSpan lowCovSpan lowCovDepth The default filtering is "2 1000 0.75 0.75 2". If you are assembling amplified data or viral data, it is possible your assembly will be flagged as unassembled. In those cases, you can turn off the filtering with the parameters :: contigFilter="2 1000 1.0 1.0 2" canu-1.3/documentation/source/pipeline.rst000066400000000000000000000007661272335422300210270ustar00rootroot00000000000000 .. _pipeline: Canu Pipeline ============= The canu pipeline is big and complicated and we haven't written it up yet. Sorry. The basic flow is: - Gatekeeper - Meryl - Overlapper + Read Correction + Read Trimming * Trimming * Splitting + Unitig Construction * Overlap Error Adjustment * Overlap Filtering (NOT IMPLEMENTED) * Unitig * Consensus * Labeling * Output Details: Meryl - counts kmers, generates a histogram txt file, and a histogram plot png canu-1.3/documentation/source/quick-start.rst000066400000000000000000000262711272335422300214700ustar00rootroot00000000000000 .. _quickstart: Canu Quick Start ================ Canu specializes in assembling PacBio or Oxford Nanopre sequences. Canu will correct the reads, then trim suspicious regions (such as remaining SMRTbell adapter), then assemble the corrected and cleaned reads into unitigs. Brief Introduction ------------------- Canu has been designed to auto-detect your resources and scale itself to fit. Two parameters let you restrict the resources used. :: maxMemory=XX maxThreads=XX Memory is specified in gigabytes. On a single machine, it will restrict Canu to at most this limit, on the grid, no single job will try to use more than the specified resources. The input sequences can be FASTA or FASTQ format, uncompressed, or compressed with gz, bz2 or xz. Running on the grid ~~~~~~~~~~~~~~~~~~~~~~ Canu is designed to run on grid environments (LSF/PBS/Torque/Slrum/SGE are supported). Currently, Canu will submit itself to the default queue with default time options. You can overwrite this behavior by providing any specific parameters you want to be used for submission as an option. Users should also specify a job name to use on the grid: :: gridOptionsJobName=myassembly "gridOptions=--partition quick --time 2:00" Assembling PacBio data ---------------------- Pacific Biosciences released P6-C4 chemistry reads. You can download them `directly `_ (7 GB) or from the `original page `_. You must have the Pac Bio SMRTpipe software installed to extract the reads as FASTQ. We made a 25X subset FASTQ available `here `_ or use the following curl command: :: curl -L -o p6.25x.fastq http://gembox.cbcb.umd.edu/mhap/raw/ecoli_p6_25x.filtered.fastq Correct, Trim and Assemble ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, canu will correct the reads, then trim the reads, then assemble the reads to unitigs. :: canu \ -p ecoli -d ecoli-auto \ genomeSize=4.8m \ -pacbio-raw p6.25x.fastq This will use the prefix 'ecoli' to name files, compute the correction task in directory 'ecoli-auto/correction', the trimming task in directory 'ecoli-auto/trimming', and the unitig construction stage in 'ecoli-auto' itself. Output files are described in the next section. Find the Output ~~~~~~~~~~~~~~~~~~~~~~ The canu progress chatter records statistics such as an input read histogram, corrected read histogram, and overlap types. Outputs from the assembly tasks are in: ecoli*/ecoli.correctedReads.fasta.gz The sequences after correction, trimmed and split based on consensus evidence. Typically >99% for PacBio and >98% for Nanopore but it can vary based on your input sequencing quality. ecoli*/ecoli.trimmedReads.fasta.gz The sequences after correction and final trimming. The corrected sequences above are overlapped again to identify any missed hairpin adapters or bad sequence that could not be detected in the raw sequences. ecoli*/ecoli.layout The layout provides information on where each read ended up in the final assembly, including contig and positions. It also includes the consensus sequence for each contig. ecoli*/ecoli.gfa The `GFA `_ is the assembly graph generated by Canu. Currently this includes the contigs, associated bubbles, and any overlaps which were not used by the assembly. The fasta output is split into three types: ecoli*/asm.contigs.fasta Everything which could be assembled and is part of the primary assembly, including both unique and repetitive elements. Each contig has several flags included on the fasta def line:: >tig######## len= reads= covStat= gappedBases= class= suggestRepeat= suggestCircular= len Length of the sequence, in bp. reads Number of reads used to form the contig. covStat The log of the ratio of the contig being unique versus being two-copy, based on the read arrival rate. Positive values indicate more likely to be unique, while negative values indicate more likely to be repetitive. See `Footnote 24 `_ in `Myers et al., A Whole-Genome Assembly of Drosophila `_. gappedBases If yes, the sequence includes all gaps in the multialignment. class Type of sequence. Unassembled sequences are primarily low-coverage sequences spanned by a single read. suggestRepeat If yes, sequence was detected as a repeat based on graph topology or read overlaps to other sequences. suggestCircular If yes, sequence is likely circular. Not implemented. ecoli*/asm.bubbles.fasta alternate paths in the graph which could not be merged into the primary assembly. ecoli*/asm.unassembled.fasta reads which could not be incorporated into the primary or bubble assemblies. Correct, Trim and Assemble, Manually ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sometimes, however, it makes sense to do the three top-level tasks by hand. This would allow trying multiple unitig construction parameters on the same set of corrected and trimmed reads. First, correct the raw reads:: canu -correct \ -p ecoli -d ecoli \ genomeSize=4.8m \ -pacbio-raw p6.25x.fastq Then, trim the output of the correction:: canu -trim \ -p ecoli -d ecoli \ genomeSize=4.8m \ -pacbio-corrected ecoli/correction/ecoli.correctedReads.fasta.gz And finally, assemble the output of trimming, twice:: canu -assemble \ -p ecoli -d ecoli-erate-0.013 \ genomeSize=4.8m \ errorRate=0.013 \ -pacbio-corrected ecoli/trimming/ecoli.trimmedReads.fasta.gz canu -assemble \ -p ecoli -d ecoli-erate-0.025 \ genomeSize=4.8m \ errorRate=0.025 \ -pacbio-corrected ecoli/trimming/ecoli.trimmedReads.fasta.gz The directory layout for correction and trimming is exactly the same as when we ran all tasks in the same command. Each unitig construction task needs its own private work space, and in there the 'correction' and 'trimming' directories are empty. The error rate always specifies the error in the corrected reads which is typically <1% for PacBio data and <2% for Nanopore data (<1% on newest chemistries). Assembling Oxford Nanopore data -------------------------------- A set of E. coli runs were released by the Loman lab. You can download one `directly `_ or any of them from the `original page `_. or use the following curl command: :: curl -L -o oxford.fasta http://nanopore.s3.climb.ac.uk/MAP006-PCR-1_2D_pass.fasta Canu assembles any of the four available datasets into a single contig but we picked one dataset to use in this tutorial. Then, assemble the data as before:: canu \ -p ecoli -d ecoli-oxford \ genomeSize=4.8m \ -nanopore-raw oxford.fasta The assembled identity is >99% before polishing. Assembling With Multiple Technologies/Files ------------------------------------------- Canu takes an arbitrary number of input files/formats. We made a mixed dataset of about 10X of a PacBio P6 and 10X of an Oxford Nanopore run available `here `_ or use the following curl command: :: curl -L -o mix.tar.gz http://gembox.cbcb.umd.edu/mhap/raw/ecoliP6Oxford.tar.gz tar xvzf mix.tar.gz Now you can assemble all the data:: canu \ -p ecoli -d ecoli-mix \ genomeSize=4.8m \ -pacbio-raw pacbio*fastq.gz \ -nanopore-raw oxford.fasta.gz .. _quick_low: Assembling Low Coverage Datasets ---------------------------------- When you have 30X or less coverage, it helps to adjust the Canu assembly parameters. Typically, assembly 20X of single-molecule data outperforms hybrid methods with higher coverage. You can download a 20X subset of `S. cerevisae `_ or use the following curl command: :: curl -L -o yeast.20x.fastq.gz http://gembox.cbcb.umd.edu/mhap/raw/yeast_filtered.20x.fastq.gz and run the assembler adding sensitive parameters (**errorRate=0.035**):: canu \ -p asm -d yeast \ genomeSize=12.1m \ errorRate=0.035 \ -pacbio-raw yeast.20x.fastq.gz After the run completes, we can check the assembly statistics:: tgStoreDump -sizes -s 12100000 -T yeast/unitigging/asm.tigStore 2 -G yeast/unitigging/asm.gkpStore :: lenSuggestRepeat sum 160297 (genomeSize 12100000) lenSuggestRepeat num 12 lenSuggestRepeat ave 13358 lenUnassembled ng10 13491 bp lg10 77 sum 1214310 bp lenUnassembled ng20 11230 bp lg20 176 sum 2424556 bp lenUnassembled ng30 9960 bp lg30 290 sum 3632411 bp lenUnassembled ng40 8986 bp lg40 418 sum 4841978 bp lenUnassembled ng50 8018 bp lg50 561 sum 6054460 bp lenUnassembled ng60 7040 bp lg60 723 sum 7266816 bp lenUnassembled ng70 6169 bp lg70 906 sum 8474192 bp lenUnassembled ng80 5479 bp lg80 1114 sum 9684981 bp lenUnassembled ng90 4787 bp lg90 1348 sum 10890099 bp lenUnassembled ng100 4043 bp lg100 1624 sum 12103239 bp lenUnassembled ng110 3323 bp lg110 1952 sum 13310167 bp lenUnassembled ng120 2499 bp lg120 2370 sum 14520362 bp lenUnassembled ng130 1435 bp lg130 2997 sum 15731198 bp lenUnassembled sum 16139888 (genomeSize 12100000) lenUnassembled num 3332 lenUnassembled ave 4843 lenContig ng10 770772 bp lg10 2 sum 1566457 bp lenContig ng20 710140 bp lg20 4 sum 3000257 bp lenContig ng30 669248 bp lg30 5 sum 3669505 bp lenContig ng40 604859 bp lg40 7 sum 4884914 bp lenContig ng50 552911 bp lg50 10 sum 6571204 bp lenContig ng60 390415 bp lg60 12 sum 7407061 bp lenContig ng70 236725 bp lg70 16 sum 8521520 bp lenContig ng80 142854 bp lg80 23 sum 9768299 bp lenContig ng90 94308 bp lg90 33 sum 10927790 bp lenContig sum 12059140 (genomeSize 12100000) lenContig num 56 lenContig ave 215341 Consensus Accuracy ------------------- While Canu corrects sequences and has 99% identity or greater with PacBio or Nanopore sequences, for the best accuracy we recommend polishing with a sequence-specific tool. We recommend `Quiver `_ for PacBio and `Nanopolish `_ for Oxford Nanpore data. If you have Illumina sequences available, `Pilon `_ can also be used to polish either PacBio or Oxford Nanopore assemblies. Futher Reading ------------------- See the `FAQ `_ page for commonly-asked questions and the `release `_. notes page for information on what's changed and known issues. canu-1.3/documentation/source/tutorial.rst000066400000000000000000000472641272335422300210710ustar00rootroot00000000000000 .. _celera-assembler: `Celera Assembler ` .. _tutorial: Canu Tutorial ============= Canu assembles reads from PacBio RS II or Oxford Nanopore MinION instruments into uniquely-assemblable contigs, unitigs. Canu owes lots of it design and code to `celera-assembler`_. Canu can be run using hardware of nearly any shape or size, anywhere from laptops to computational grids with thousands of nodes. Obviouisly, larger assemblies will take a long time to compute on laptops, and smaller assemblies can't take advantage of hundreds of nodes, so what is being assembled plays some part in determining what hardware can be effectively used. Most algorithms in canu have been multi-threaded (to use all the cores on a single node), parallelized (to use all the nodes in a grid), or both (all the cores on all the nodes). .. _canu-command: Canu, the command ~~~~~~~~~~~~~~~~~~~~~~ The **canu** command is the 'executive' program that runs all modules of the assembler. It oversees each of the three top-level tasks (correction, trimming, unitig construction), each of which consists of many steps. Canu ensures that input files for each step exist, that each step successfully finished, and that the output for each step exists. It does minor bits of processing, such as reformatting files, but generally generally just executes other programs. :: canu \ -d \ -p \ [-s specifications] \ [-correct | -trim | -assemble] \ errorRate= \ genomeSize=\ [parameters] \ [-pacbio-raw ] [-pacbio-corrected ] [-nanopore-raw ] [-nanopore-corrected ] Two options are mandatory, -d, to set the working directory, and -p, to set the file name prefix. All work is performed and output appears in the working directory. The directory need not exist before starting. Most files in this directory have file names beginning with the file name prefix, however, running two canu commands in the same directory will probably lead to confusion. The -s option will import a list of parameters from the supplied specification ('spec') file. These parameters will be applied before any from the command line are used, providing a method for setting commonly used parameters, but overriding them for specific assemblies. By default, all three top-level tasks are performed. It is possible to run exactly one task by using the -correct, -trim or -assemble options. These options can be useful if you want to correct reads once and try many different assemblies. We do exactly that in the :ref:`quickstart`. Parameters are key=value pairs that configure the assembler. They set run time parameters (e.g., memory, threads, grid), algorithmic parameters (e.g., error rates, trimming aggressiveness), and enable or disable entire processing steps (e.g., don't correct errors, don't search for subreads). They are described later. One parameter is required: the genomeSize (in bases, with common SI prefixes allowed, for example, 4.7m or 2.8g; see :ref:`genomeSize`). Parameters are listed in the :ref:`parameter-reference`, but the common ones are described in this document. Reads are supplied to canu by options that options that describe how the reads were generated, and what level of quality they are, for example, -pacbio-raw indicates the reads were generated on a PacBio RS II instrument, and have had no processing done to them. Each file of reads supplied this way becomes a 'library' of reads. The reads should have been (physically) generated all at the same time using the same steps, but perhaps sequenced in multiple batches. In canu, each library has a set of options setting various algorithmic parameters, for example, how aggressively to trim. To explicitly set library parameters, a text 'gkp' file describing the library and the input files must be created. Don't worry too much about this yet, it's an advanced feature, fully described in Section :ref:`gkp-files`. The read-files contain sequence data in either FASTA or FASTQ format (or both! A quirk of the implementation allows files that contain both FASTA and FASTQ format reads). The files can be uncompressed, gzip, bzip2 or xz compressed. We've found that "gzip -1" provides good compression that is fast to both compress and decompress. For 'archival' purposes, we use "xz -9". .. _canu-pipeline: Canu, the pipeline ~~~~~~~~~~~~~~~~~~~~~~ The canu pipeline, that is, what it actually computes, comprises of computing overlaps and processing the overlaps to some result. Each of the three tasks (read correction, read trimming and unitig construction) follow the same pattern: * Load reads into the read database, gkpStore. * Compute k-mer counts in preparation for the overlap computation. * Compute overlaps. * Load overlaps into the overlap database, ovlStore. * Do something interesting with the reads and overlaps. * The read correction task will replace the original noisy read sequences with consensus sequences computed from overlapping reads. * The read trimming task will use overlapping reads to decide what regions of each read are high-quality sequence, and what regions should be trimmed. After trimming, the single largest high-quality chunk of sequence is retained. * The unitig construction task finds sets of overlaps that are consistent, and uses those to place reads into a multialignment layout. The layout is then used to generate a consensus sequence for the unitig. .. _module-tags: Module Tags ~~~~~~~~~~~~~~~~~~~~~~ Because each of the three tasks share common algorithms (all compute overlaps, two compute consensus sequences, etc), parameters are differentiated by a short prefix 'tag' string. This lets canu have one generic parameter that can be set to different values for each stage in each task. For example, "corOvlMemory" will set memory usage for overlaps being generated for read correction; "obtOvlMemory" for overlaps generated for Overlap Based Trimming; "utgOvlMemory" for overlaps generated for unitig construction. The tags are: +--------+-------------------------------------------------------------------+ |Tag | Usage | +========+===================================================================+ |master | the canu script itself, and any components that it runs directly | +--------+-------------------------------------------------------------------+ +--------+-------------------------------------------------------------------+ |cns | unitig consensus generation | +--------+-------------------------------------------------------------------+ |cor | read correction generation | +--------+-------------------------------------------------------------------+ +--------+-------------------------------------------------------------------+ |red | read error detection | +--------+-------------------------------------------------------------------+ |oea | overlap error adjustment | +--------+-------------------------------------------------------------------+ +--------+-------------------------------------------------------------------+ |ovl | the standard overlapper | +--------+-------------------------------------------------------------------+ |corovl | the standard overlapper, as used in the correction phase | +--------+-------------------------------------------------------------------+ |obtovl | the standard overlapper, as used in the trimming phase | +--------+-------------------------------------------------------------------+ |utgovl | the standard overlapper, as used in the assembly phase | +--------+-------------------------------------------------------------------+ +--------+-------------------------------------------------------------------+ |mhap | the mhap overlapper | +--------+-------------------------------------------------------------------+ |cormhap | the mhap overlapper, as used in the correction phase | +--------+-------------------------------------------------------------------+ |obtmhap | the mhap overlapper, as used in the trimming phase | +--------+-------------------------------------------------------------------+ |utgmhap | the mhap overlapper, as used in the assembly phase | +--------+-------------------------------------------------------------------+ +--------+-------------------------------------------------------------------+ |mmap | the `minimap `_ overlapper | +--------+-------------------------------------------------------------------+ |cormmap | the minimap overlapper, as used in the correction phase | +--------+-------------------------------------------------------------------+ |obtmmap | the minimap overlapper, as used in the trimming phase | +--------+-------------------------------------------------------------------+ |utgmmap | the minimap overlapper, as used in the assembly phase | +--------+-------------------------------------------------------------------+ +--------+-------------------------------------------------------------------+ |ovb | the bucketizing phase of overlap store building | +--------+-------------------------------------------------------------------+ |ovs | the sort phase of overlap store building | +--------+-------------------------------------------------------------------+ We'll get to the details eventually. .. _execution: Execution Configuration ~~~~~~~~~~~~~~~~~~~~~~~~ There are two modes that canu runs in: locally, using just one machine, or grid-enabled, using multiple hosts managed by a grid engine. At present, only Sun Grid Engine / Open Grid Engine / Univa Grid Engine / SGE / whatever it's called now, PBS, and Slurm support is available. LSF support is also included but has limited testing. Section :ref:`grid-engine-config` has a few hints on how to set up a new grid engine. To enable execution of the parallel steps on the grid, set ``useGrid=1``. The canu pipeline will immediately submit itself to the grid, and run entirely under grid control. This is the default if a grid-engine is detected on your system. If you prefer to run locally, set ``useGrid=0``. In both cases, local or grid, Canu will auto-detect available resources and scale the jobs to run, based on the resources and genome size you're assembling. Thus, most users should be able to run the command without modifying the defaults. Some advanced options are outlined below. Each stage has the same five configuration options, and tags are used to specialize the option to a specific stage. The options are: useGrid=boolean Run this stage on the grid, usually in parallel. gridOptions=string Supply this string to the grid submit command. Memory=integer Use this many gigabytes of memory, per process. Threads Use this many compute threads per process. Concurrency If not on the grid, run this many jobs at the same time. Global grid options, applied to every job submitted to the grid, can be set with 'gridOptions'. This can be used to add accounting information or access credentials. A name can be associated with this compute using 'gridOptionsJobName'. Canu will work just fine with no name set, but if multiple canu assemblies are running at the same time, they will tend to wait for each others jobs to finish. For example, if two assemblies are running, at some point both will have overlap jobs running. Each assembly will be waiting for all jobs named 'ovl_asm' to finish. Had the assemblies specified job names, gridOptionsJobName=apple and gridOptionsJobName=orange, then one would be waiting for jobs named 'ovl_asm_apple', and the other would be waiting for jobs named 'ovl_asm_orange'. .. _error-rates: Error Rates ~~~~~~~~~~~~~~~~~~~~~~ Canu expects all error rates to be reported as fraction error, not as percent error. We're not sure exactly why this is so. Previously, it used a mix of fraction error and percent error (or both!), and was a little confusing. Here's a handy table you can print out that converts between fraction error and percent error. Not all values are shown (it'd be quite a large table) but we have every confidence you can figure out the missing values: ============== ============= Fraction Error Percent Error ============== ============= 0.01 1% 0.02 2% 0.03 3% . . . . 0.12 12% . . . . ============== ============= Eventually, we want to have Canu take a single error rate, the error rate of a single input read, and derive all other rates from there. This is the parameter ``errorRate``. Currently, the defaults are 0.025 for PacBio sequences and 0.05 for Oxford Nanpore sequences. Typically, you should not need to modify this setting. However, the error rate does affect runtime and lowering it can significantly speed up your assembly. Thus, for low coverage datasets (<=30X) we recommend increasing the error rate slightly (by 1%, so errorRate=0.035 or PacBio) and for high-coverage (>=60X) datasets, we recommend decreasing it (by 1%, so errorRate=0.015 for PacBio). The following error rates are defined: errorRate The expected error rate in a corrected single read. This will set the remaining error rates implicitly. Recall there are three sets of overlaps generated: one for read correction, one for read trimming, and one for unitig construction. The read correction overlaps are typically generated by the 'mhap' overlapper, which doesn't need an error rate as it doesn't generate alignment based overlaps. The other two sets of overlaps are generated by overlapInCore, which is alignment based, and so requires a maximum error rate. Thus, out of the six possible tags, only two are currently useful for overlap generation: obtOvlErrorRate Do not compute overlaps used for trimming above this error rate. Applies to the standard overlapper, and realigning mhap overlaps. utgOvlErrorRate Do not compute overlaps used for unitig construction above this error rate. Applies to the standard overlapper, and realigning mhap overlaps. (ADVANCED) It is possible to convert the mhap or minimap overlaps to alignment based overlaps using ``obtReAlign=true`` or ``ovlReAlign=true``. If so, the overlaps will be computed using either ``obtOvlErrorRate`` or ``utgOvlErrorRate``, depending on which overlaps are being generated. Be sure to not confuse ``obtOvlErrorRate`` with ``obtErrorRate``: obtErrorRate Filter overlaps during OBT's read trimming and read splitting. In `celera-assembler`_, consensus generation required an overlap. In canu, this is no longer used, but the ``cnsErrorRate`` option still exists. .. _minimum-lengths: Minimum Lengths ~~~~~~~~~~~~~~~~~~~~~~ Two minimum sizes are known: minReadLength Discard reads shorter than this when loading into the assembler, and when trimming reads. minOverlapLength Do not save overlaps shorter than this. Overlap configuration ~~~~~~~~~~~~~~~~~~~~~~ The largest compute of the assembler is also the most complicated to configure. As shown in the 'module tags' section, there are up to eight (!) different overlapper configurations. For each overlapper ('ovl' or 'mhap') there is a global configuration, and three specializations that apply to each stage in the pipeline (correction, trimming or assembly). Like with 'grid configuration', overlap configuration uses a 'tag' prefix applied to each option. The tags in this instance are 'cor', 'obt' and 'utg'. For example: - To change the k-mer size for all instances of the ovl overlapper, 'merSize=23' would be used. - To change the k-mer size for just the ovl overlapper used during correction, 'corMerSize=16' would be used. - To change the mhap k-mer size for all instances, 'mhapMerSize=18' would be used. - To change the mhap k-mer size just during correction, 'corMhapMerSize=15' would be used. - To use minimap for overlap computation just during correction, 'corOverlapper=minimap' would be used. Ovl Overlapper Configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Overlapper select the overlap algorithm to use, 'ovl' or 'mhap'. Ovl Overlapper Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~ ovlHashBlockLength how many bases to reads to include in the hash table; directly controls process size ovlRefBlockSize how many reads to compute overlaps for in one process; directly controls process time ovlRefBlockLength same, but use 'bases in reads' instead of 'number of reads' ovlHashBits size of the hash table (SHOULD BE REMOVED AND COMPUTED, MAYBE TWO PASS) ovlHashLoad how much to fill the hash table before computing overlaps (SHOULD BE REMOVED) ovlMerSize size of kmer seed; smaller - more sensitive, but slower The overlapper will not use frequent kmers to seed overlaps. These are computed by the 'meryl' program, and can be selected in one of three ways. Terminology. A k-mer is a contiguous sequence of k bases. The read 'ACTTA' has two 4-mers: ACTT and CTTA. To account for reverse-complement sequence, a 'canonical kmer' is the lexicographically smaller of the forward and reverse-complemented kmer sequence. Kmer ACTT, with reverse complement AAGT, has a canonical kmer AAGT. Kmer CTTA, reverse-complement TAAG, has canonical kmer CTTA. A 'distinct' kmer is the kmer sequence with no count associated with it. A 'total' kmer (for lack of a better term) is the kmer with its count. The sequence CGTTTTTTTCGTCG has (forward) 12 'total' 4-mers and 7 'distinct' kmers. :: TCGTTTTTTTCGTCG count TCGT 2 distinct-1 CGTT 1 distinct-2 GTTT 1 distinct-3 TTTT 4 distinct-4 TTTT 4 copy of distinct-4 TTTT 4 copy of distinct-4 TTTT 4 copy of distinct-4 TTTC 4 copy of distinct-4 TTCG 1 distinct-5 TCGT 2 copy of distinct-1 CGTC 1 distinct-6 GTCG 1 distinct-7 MerThreshold any kmer with count higher than N is not used MerDistinct pick a threshold so as to seed overlaps using this fraction of all distinct kmers in the input. In the example above, fraction 0.8572 of the k-mers (6/7) will be at or below threshold 2. MerTotal pick a threshold so as to seed overlaps using this fraction of all kmers in the input. In the example above, fraction 0.6364 of the k-mers (7/11) will be at or below threshold 2. FrequentMers don't compute frequent kmers, use those listed in this fasta file Mhap Overlapper Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~ MhapBlockSize Chunk of reads that can fit into 1GB of memory. Combined with memory to compute the size of chunk the reads are split into. MhapMerSize Use k-mers of this size for detecting overlaps. ReAlign After computing overlaps with mhap, compute a sequence alignment for each overlap. MhapSensitivity Either 'normal', 'high', or 'fast'. Mhap also will down-weight frequent kmers (using tf-idf), but it's selection of frequent is not exposed. Minimap Overlapper Parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~ MMapBlockSize Chunk of reads that can fit into 1GB of memory. Combined with memory to compute the size of chunk the reads are split into. MMapMerSize Use k-mers of this size for detecting overlaps Minimap also will ignore high-frequency minimzers, but it's selection of frequent is not exposed. canu-1.3/logo.jpg000066400000000000000000001264711272335422300137630ustar00rootroot00000000000000ÿØÿàJFIFHHÿáExifMM*nv(1"~2 ‡i´,,Adobe Illustrator CS6 (Macintosh)2015:08:21 19:34:15ê  È ž2015:08:21 15:34:11ÿá>ïhttp://ns.adobe.com/xap/1.0/ ÿídPhotoshop 3.08BIM,Z%G>20150821? 153411-04008BIM%ð©àûµžÇÜ„ÚOx %Çÿâ XICC_PROFILE HLinomntrRGB XYZ Î 1acspMSFTIEC sRGBöÖÓ-HP cprtP3desc„lwtptðbkptrXYZgXYZ,bXYZ@dmndTpdmddĈvuedL†viewÔ$lumiømeas $tech0 rTRC< gTRC< bTRC< textCopyright (c) 1998 Hewlett-Packard CompanydescsRGB IEC61966-2.1sRGB IEC61966-2.1XYZ óQÌXYZ XYZ o¢8õXYZ b™·…ÚXYZ $ „¶ÏdescIEC http://www.iec.chIEC http://www.iec.chdesc.IEC 61966-2.1 Default RGB colour space - sRGB.IEC 61966-2.1 Default RGB colour space - sRGBdesc,Reference Viewing Condition in IEC61966-2.1,Reference Viewing Condition in IEC61966-2.1view¤þ_.ÏíÌ \žXYZ L VPWçmeassig CRT curv #(-27;@EJOTY^chmrw|†‹•šŸ¤©®²·¼ÁÆËÐÕÛàåëðöû %+28>ELRY`gnu|ƒ‹’š¡©±¹ÁÉÑÙáéòú &/8AKT]gqz„Ž˜¢¬¶ÁËÕàëõ !-8COZfr~Š–¢®ºÇÓàìù -;HUcq~Œš¨¶ÄÓáðþ +:IXgw†–¦µÅÕåö'7HYj{Œ¯ÀÑãõ+=Oat†™¬¿Òåø 2FZn‚–ª¾Òçû  % : O d y ¤ º Ï å û  ' = T j ˜ ® Å Ü ó " 9 Q i € ˜ ° È á ù  * C \ u Ž § À Ù ó & @ Z t Ž © Ã Þ ø.Id›¶Òî %A^z–³Ïì &Ca~›¹×õ1OmŒªÉè&Ed„£Ãã#Ccƒ¤Åå'Ij‹­Îð4Vx›½à&Il²ÖúAe‰®Ò÷@eŠ¯Õú Ek‘·Ý*QwžÅì;cŠ²Ú*R{£ÌõGp™Ãì@j”¾é>i”¿ê  A l ˜ Ä ð!!H!u!¡!Î!û"'"U"‚"¯"Ý# #8#f#”#Â#ð$$M$|$«$Ú% %8%h%—%Ç%÷&'&W&‡&·&è''I'z'«'Ü( (?(q(¢(Ô))8)k))Ð**5*h*›*Ï++6+i++Ñ,,9,n,¢,×- -A-v-«-á..L.‚.·.î/$/Z/‘/Ç/þ050l0¤0Û11J1‚1º1ò2*2c2›2Ô3 3F33¸3ñ4+4e4ž4Ø55M5‡5Â5ý676r6®6é7$7`7œ7×88P8Œ8È99B99¼9ù:6:t:²:ï;-;k;ª;è<' >`> >à?!?a?¢?â@#@d@¦@çA)AjA¬AîB0BrBµB÷C:C}CÀDDGDŠDÎEEUEšEÞF"FgF«FðG5G{GÀHHKH‘H×IIcI©IðJ7J}JÄK KSKšKâL*LrLºMMJM“MÜN%NnN·OOIO“OÝP'PqP»QQPQ›QæR1R|RÇSS_SªSöTBTTÛU(UuUÂVV\V©V÷WDW’WàX/X}XËYYiY¸ZZVZ¦Zõ[E[•[å\5\†\Ö]']x]É^^l^½__a_³``W`ª`üaOa¢aõbIbœbðcCc—cëd@d”dée=e’eçf=f’fèg=g“géh?h–hìiCišiñjHjŸj÷kOk§kÿlWl¯mm`m¹nnknÄooxoÑp+p†pàq:q•qðrKr¦ss]s¸ttptÌu(u…uáv>v›vøwVw³xxnxÌy*y‰yçzFz¥{{c{Â|!||á}A}¡~~b~Â#„å€G€¨ kÍ‚0‚’‚ôƒWƒº„„€„ã…G…«††r†×‡;‡ŸˆˆiˆÎ‰3‰™‰þŠdŠÊ‹0‹–‹üŒcŒÊ1˜ÿŽfŽÎ6žnÖ‘?‘¨’’z’ã“M“¶” ”Š”ô•_•É–4–Ÿ— —u—à˜L˜¸™$™™üšhšÕ›B›¯œœ‰œ÷dÒž@ž®ŸŸ‹Ÿú i Ø¡G¡¶¢&¢–££v£æ¤V¤Ç¥8¥©¦¦‹¦ý§n§à¨R¨Ä©7©©ªª««u«é¬\¬Ð­D­¸®-®¡¯¯‹°°u°ê±`±Ö²K²Â³8³®´%´œµµŠ¶¶y¶ð·h·à¸Y¸Ñ¹J¹Âº;ºµ».»§¼!¼›½½¾ ¾„¾ÿ¿z¿õÀpÀìÁgÁãÂ_ÂÛÃXÃÔÄQÄÎÅKÅÈÆFÆÃÇAÇ¿È=ȼÉ:ɹÊ8Ê·Ë6˶Ì5̵Í5͵Î6ζÏ7ϸÐ9кÑ<ѾÒ?ÒÁÓDÓÆÔIÔËÕNÕÑÖUÖØ×\×àØdØèÙlÙñÚvÚûÛ€ÜÜŠÝÝ–ÞÞ¢ß)߯à6à½áDáÌâSâÛãcãëäsäüå„æ æ–çç©è2è¼éFéÐê[êåëpëûì†ííœî(î´ï@ïÌðXðåñrñÿòŒóó§ô4ôÂõPõÞömöû÷Šøø¨ù8ùÇúWúçûwüü˜ý)ýºþKþÜÿmÿÿÿÀžÈÿÄ ÿĵ}!1AQa"q2‘¡#B±ÁRÑð$3br‚ %&'()*456789:CDEFGHIJSTUVWXYZcdefghijstuvwxyzƒ„…†‡ˆ‰Š’“”•–—˜™š¢£¤¥¦§¨©ª²³´µ¶·¸¹ºÂÃÄÅÆÇÈÉÊÒÓÔÕÖ×ØÙÚáâãäåæçèéêñòóôõö÷øùúÿÄ ÿĵw!1AQaq"2B‘¡±Á #3RðbrÑ $4á%ñ&'()*56789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz‚ƒ„…†‡ˆ‰Š’“”•–—˜™š¢£¤¥¦§¨©ª²³´µ¶·¸¹ºÂÃÄÅÆÇÈÉÊÒÓÔÕÖ×ØÙÚâãäåæçèéêòóôõö÷øùúÿÛCÿÛCÿÝÿÚ ?þêõ?‡º{(¥ð§‡öù³¾HÓ‡+m&9û? p ÆyÀÜ*tÚi–­˸zmËÞ[Ým×s‹F38Ù¯Þ6ùuoÜŸª]Ã÷ꊖ¿ ük¨[´~Ð~x/ÑôãŒbg`ôÎO<6Š(Ežö²¥NÝ÷²¿Mù}æX|Q×3|N»wUoºÕJéËå×[ꥫ*Ið—À77ò·†4!›¢¸>œ6…·µÈÀ·=ÏLŒbEÊœµöp½î×$?d­Þ:¦÷‹½£Í[ ÏZ¤Ój<ê6²½ÔbÝíöºïÛvhé |§–|-¡6Ë»À húq8’€îà0Nl(Ý8GNJzw§þ÷ý.ôZhva©%F1r”œeR-Þ×å©(÷zÙv]Öæ]¿Â?‡Ó[‡ h¥¤3±ÿ‰M€äÏ)#ˆ»ç ÀþöÓØRz¨C[ïN7²èÛÝ^Ú{ÉYhíhð¼*’rævmßE}—£};-ÝþjÛ|5ð hЯü">mšrrúE‰;…·R|¬îÏS’}AÆ){:wåä…¯o‚7JÿÍfïÛWµ“Z:QT"·µ«»ô†š&ú®ýu±“'‡Á|áÒËg¹KiyÝäÎ|¼žyäŽ}rE“z ۥ“¾ß-ݺêÖÊ÷n\Â%ÊüÏ•6üùoweu¿–»·cwPøiðö[ø+Âøt‰[œ§H†F ã<‚=ýÍ z{:núk¶ÿíësäÑqéV„gNi­ãÊ­¥¿?Ëï3¥øKðâ ™<á VöÔäh¶$Ü'_Üàú`œ¼{Jþä[x/K]ÞéoËkm£ÕT*RšZF¥%vô÷¤•¾µ£ï·TÚ-j þM¼_ð†xmß[ä¦`­¼Ÿ›ÊÈÎ÷½êÔ*tÞŽœî£ÛÕ­Zò¿þ£:qTÕJj6ÖU)¤ß”¯¶·û¼î¬g§Â?‡V÷vn¾ðøÝ4¡ÁÒ,J06³àåœàóœ‘Ó´{*kzqÑ^ÖVZÿ/½yZßÝoß% 7-H;»&îí«÷ed¶[êÓz¨ÝXÞ? þnðo‡8Ñ´ó´vÇî{vèÚã ¹!ü±^iE|“ViyZÞ‡¤áÓeeåé·ç¯–ƒáV|=Æ?áðî9ÿ˜&™ßÐ}›üsß8Í‘Û–+Ñ%§k®ŸÝ½·z\|©m¦·ó½­[7~Üß6|# m[HÑ¢éx$Ës“Í°ùŽõ ’¡E5NräŽTU—¤4àÒí#‚¾š¢PoX-4ß™÷‹I[_µò±>™ðŸáõ»ß ðŸ‡Ïï­ðN‰¥îÚ¡Ú3h@îOÉÛÒg•£µ{òŽúIÝ«.Î?;ÈÓ ECÚEÞéÇFûÁ_¤um_g«Ór›üø{qu'ü"^é!:—Œ-´ sþŒÜÇqÈëÏ94*pþZnýe6¾rMýÊÏËS¸Vç6›øš}-óå{Ývôw±wIøSðþ(%ŒøGÄÇ{x¹þÂÒvTslrp€çù ’÷!ßÞ„dïäÜn¶éË÷ÞFøZiRäméR¦›ý·¿Ã¯ßèõr¡ÂÌd“þ?m’æìt=0à}¦AÔ[àŒt s€0«B¥ä¦ïmeÉê·¼®ôò^—w8êaç)ÎQwýåK½4Jm'ðÉÙE-º'{éË©§ü-ø~še·üQþ$YíÉд¬¨À~ÊÄœó“Û<í\W\²ÓXEí×TýçÖNÍõzqT¡ö›‚ÕÝtzYtô^z^FWü*‡£O ÿ‡ 6;þÃÒþ÷Ù·d³î'999õînœK’¯Ò1M[¼­Ì¯×Þmõ¾¬ó\ŠI¾e §¢Úß•Þí_¥õѤѷwðÓáøҦǃ<0 ´ ö—YqþŠ=³œúæ—$—$m~Êÿ¢»ë.[½–‡¥V Ø5µ©èû{º4—_ËæÏÿÐþøn9º³»\ÿé1íÎsúÛ­4ßGk´¿=Õ×åÕ;™TW•4ör—þ‘/øqþ›jpAònóõ&Û¯å߯SÍ6ÛŠ¿~‘µ·Ò÷iü’ù„#ËUéÊù$Ò½ôv½’µºjüž¶÷‰út|õµœ gµÅ¹ÉéÎr:0è~nŠ–Ïå×ôÒÿ§ìW©MöŒÕì¶rƒ·á}¤µòå‘Íûu²LP.fíÏNç#ž)·Ã_¿GÕµµþ}Y4ïídíkÓm}ùýÞ—ûî†(Íåç]ž{c䟎F÷ÕGD¯Ùõ·]|ïÛ_ºäÔNUf­î¨Rûùegé¯U$>Ðb]Cœæh¾>ÉéÏÇåS-ÿ­ÿ^×ÓUäÜ®…’©Ñ¹Å¿üy+ok]ü¬„„÷˜ÿ¦8êFG‘kíÇ>›²©œ]­×ðjÏ£z;úùid)BRk6¿xï}¬áOKiµ¯~½/¢$±C #í—yÿÀ¹yïרç¿z™;¶ûÿ^_-=lUÔ,ÿž¯þžÞ^­ú²;ÑqЇNââlÿn}~½ÕÞÎýV–Û¥»;tÒÞNöLšpý×~e+YoïÉ­:½·IéÒé2>ãì+øæãëéøáK~úýúùë÷—à­ÿƒÿ¶_ÒUÙ²Gú ãÓýÄvï·ŽØÎiÅ«®÷üþõ¿ù+nK_¸M·¥(Éÿà¸ß¯e¦ÞŽÃï,ÜŽËÿÇàôöÿ<Ò[®ºùÿÀõÒÿš.¯ðç§géiÅßÑ»ûÿ×í·þ”.?_óÖš[¯/]žÛ­tóôØUvŽŸòò›·ý¾šðüÖœãý:ÇýhÇéØóÔcšK¯§ê¿¯øaÔzCþ¾Ã§ø¿o×Dýbé×l¶xà}¥¿/³\qß¿=½9Å izmó_—êKš­ñ¿/±Sõÿ>¬¹õÿ?äÒ,n>läõ#¹çÔŽƒÐ}MRuäÀ`¢ÛsÏ?¼¹¼uãåCÉ[NÑ~o¢¿MoëÓ®ýŒ¥êm§"Õ÷æ~keg®û+jâ[ô‹áœåíóÇCöU` gðãÑ”ºk}7ëóZÛïóÖ⦚[÷†Û|=6ü´ò%ñµ±éßì¶ãè/®ìIôùo×úé}ÑJ7r³Ó™ß®­%åÓÓ^ÿ¶CçÞúèÿäaþ÷ôü:«}=? {ßeç÷…%hË»­ZÿøõÒ:ôÐ-˜ÉÈâæì{óu7û¹÷à{qÍ;ü:t]ß[t·nÏ·~|ã5&Ÿ-§[ð”úé¿kÛËq,¸ÓâïþÛþþzŸ¯]²÷ùïý]þyµ;ªt¯º§ïÞÝ-~»k¿uq¡GönyÇØxïÖ×òè{ÍUÒ•üß^ž^½í/Mˆ”±²º½;®¿eÚþò^¶oð¸\àé’c86È}ñ±8é‚?¨©¿^»üÿR§­wt£ø¨ú^ËÓkécÿÑþñöömUV -/mEÕÚB¬2Ãå‚x<Á’óBÉ —Í]ÊJô~«õÿ?êÆs¿=4ïéìç¯è÷ýc¯Ü7—RÂÙSòHŒ­±JŸgÝÑ7ï!•8ÊH¡ˆe# ÊÌþÏý½ú~–üFÛö‘xÉùèã§âÞÝöº,2ƒGc¸ýg·ÏùÿIÙm­×šëßKü¾ë «Î=¹dÝûÞ6Ó®î j$ú ÷Ïü¼Ëôëœû{ç4ïtÿ­/÷jÝ÷û‰I*®Ë_gôûS·Ý¯ßæ,@}®ëýËAžr~Y±Ó§çÎp3Í.‹Õþƒ_Ũí¯-.Úé/_éëmŒã¨ù3ë+K»Ók=šÍ’$³5´;Œq¹E”Á黎eû@L‚FxUÎß?ë·N‹¯]/Þ¨¿¾ŸÏ’|¶~"Ö™säwSÛ¾ød»ÆÅd¸†Û!ã‘VXX•’9Q]mdR Ðúy/Õÿ_ð†õz?hüíîBßwü„ö\Å.p?Óo>Ÿñ÷/ׯÓéšùyôþ½6è¾ÿ_*ÿéÙíþ_!l@«éºpzŽòöä÷õç¯4=4þ¿Oë¾ã¥ü8ÿÛß/~Z[ø–EÓÈcÿž$_Ô-ïZýw_ðûwö_‹‡£ëå÷”IJO_°1ëÍ·½ž1ž½3è([¯Uýtüþâ_û»ëûŸ¿÷~vüWÜ-àc“Ô,`õë¾.?ï×ß*/‰zéÒúôÛóû‹¨Ò„¯µ—æ´é×O]Uíqo†b‹¶/-sÿ/1õé×ÿÕ»3êíç­¿á­u~š_et¨¢­¯µ§ÿ¥wÓþ ºÚÁv2m}¯ =ÿé§_n{…%×Ó^šoúmþaQ]CʬôÓ~þ¾{y§\­²ÿ¯–õÿŸkóψ.¾Ÿ¯ãøwéi9oñ?ý"×OÒVøçù…"¿¯ëúüP5é³sÿ.¶ã<~òçè}8Ï~Üš}/æ¿«ZÏïÓ¶·'íùò-:|O_^?7$·é7ßïÛé>8ü>˜ýX}—å§Îö×oÖSŽ§¬?ôþ-¼Ó’ÛýûìÿÏÑÇþ[ñïßðǽ [óûþÿë¶Ã‡ÛÕüoä­ç×”[gô®y÷cƒÌR=ú÷'ÐñMô¿o×OÀTÝÔõ¿ïj½­kËþu™ÌN=.®Çôº“œãÇ>ÇŒ©ü·ì¼úöýï¾í¨”ôŒ›þzÏm½ù|›^¯ÕØŽÌìøÀãýö#ë>¿§é’i=ß«ü˧¤!§ØýM-ø÷ë`þAhéÀ\±ý;ÿ_j樂qòåטüýÍþ{ê6|e¶çÑ?D\~´¿¯ëë¶àÿ‚ïÿ>㾺Z-/[#ÿÒþìßú´z„oq¯êZ}ÑÉRÖh¢–{§bpí³H‘íc+€¨åŸGòþ¶‹_;‘/Ž—ø§ÿ¦åþ]ºë}†úÅ./lfŠI-.ü»€/-–/´lA ˆ$óÒkw'ª~V&6IBºªü¿Pk÷”ÿÃS]6|«mz¥k-õwWEf¿žÆúÕQB=³Fº…´llüÉ$œ\Å™$±Š7yK@ë™%–˜èýWëÿ{~’mûÑè¹g®áæ·Û®ÞˆÕŽHæ½Dë$fÄ’7Ys 8‘ VŒpz†Î3HKø§qyý©y/Íüšl¹êA[1øb\çê:}3O¢õ×ôþë¢þí-~SÓ§åêÞ†V€Æho®î–âþi%“4¨[FR xb ·”z·ÎÈqûw×ßZÿÛ‹É~Ÿ=Ë1Æ »ººL/yå]®â<Ð-¢ò¢m`×(åb৙Ž[ÌhÓk}=5ûßéýh(¯z¦{ýwv„7ÕþKçbå&ÇõÞqÿ_R~ç¨ýB¯ëúõÜT´‹ÿ¯µ_—Ç·ùï¿Ê+eÿ£×tÿ¤òÿ,úwïMÝ¿?ëOëç¸á~HßûÛyN_×ôÈ­Æ4Èý>Ä£ÛýQíÉïÜþx4þ×ý½ú“à¯úõÿ¶|ÿ?[‹.™!ÿ§lÁÛ§ê?,ÒŽÿÖý;õòûÄÚö§od®–öäWßçqoqöI?Ýèy#ùP›ºï~åU^䯮‰þ)þôÛ½”}Ä1ñÏÚí1øÝEïøõúQÕôþ¶ó*oNmýènº¹(üµwOïê`¢ûÞAørþäu'éêsBòìÿ¯Ô$ì¢úûJ×]ý<õ°ëõÖ]ãå»õe¸ÿëuïõ¡uôýWõÿ…-áþ/ý²e¯óÿë=ÿ/®2)/ùÿ>¼ž(šÿÇì¸ÿŸ[§úÙñßœuàŸÆëïÿ†þ®Gü¼à_úT‚ ý¦ûýûnŸõÀgŒëNM7ÓÓõ­ÿ«„W¿QùÇÿIÓúë÷·æ[þO\qùtƒëÜžr3é“CV·šMÿ_$ú[åyøªyM[Ë÷tßæÛùúž3v@Çú}Þxî2O_SøvÝÀ¡ÿ—ãývùËìªv\ö¿ñ'÷·wo›þ®-Tž×W˜íÿ/2þ¿OË"ž©®»måÙ÷駧r¡ð¾žõOÆr¿}ÿàYl2Èf {Ûàþ;é=ß«'zTÛß—×U¦›iòóvi‚È5; ~ }ýIïýëG_Ÿ§ü0’µ$¿éÚÿÒ>žn$øþËnr>Èœúä¿—ù4zWÿƒ/úô–Ò]4ÞÖ¶¶¿•ÏÿÓþð<¡pºŠ"´’.½ªËe°â 8ž(‰ §Ê‘†”íb€V—ÂýWëýtü–søéŠ_úDŒu×ïõ-FÒÇK¶wVVs=ÛÈòXÙ šT¶dº¸0Ê@{qöÛX­—tÈQ‹Gµ–—Ožÿ×o^½l;¿i7¤$»]?^¯õÑ¢Ž¡áoO|­c­ië%½Ë˜ïôëNDI‘-¥‰o&‘®÷sͲ:!ü•5tùöþ¶¿ãqN7œmÒ3ßÖ¢_{×Dsé>5ðåÝÜ@é«0}®ÏV°Ôu;'Ú;Ý2å.lìÔ¹û3£ ákxÌ–ÿfb¯ëúìL"ÕI]˸Y­÷çç-¿«è£éþ×ñŒZ„:f©äZ+YêiÕÏ’òÅö»w·f´»†áŠ:ÉfìƒÍ ±~VgÓæÿOëo¼µüYÿ‚Ÿþßùmÿ ^ÒlçÔôÆÎb¸óíÜð'‚â1;s÷DÐÌÓG$ÎR‚i6‰•_×ôÿ5Õ¾Ô_«p‹¿ß{è¾ZsKrþ\·í[-Ao\nÁh­ †ITJêq´a™6Ÿ§õýÀøªÿm®ï_ÎÖlµ¥J—‹:+ žy¥ÚäoO6fbŽ6®¥¶º0ܬ *­ïò_‘0ø_øêér¿åçù(Ëcƒj§¡&oâÿ¦òû {¯=ß“·Ý÷þxSÒ[¥Ì»}¹zþy8þËN>ÅÁÈÿž'ž‡¶?úü†ĽWæÖšéz_‡'šíÝ|º øþÍqÏüy{ ÿàgúÇžq†#¿Þÿeoc­íìÒºµ×¹eeµ×NâÞ…’pq¶O©ôî6µ×Óõ_×ü8KxyOÿlŸõ¿Þ[ÿëôëølRþ¿­ÿ?¼ ãÓ#>½±ôõôüûP8Èûtò[Oý4¸úÞŸ•ËÇþÿ¥KüÆÁÍÍðá­{ƒ’mÏO”pG\çp:U=£¿_ϧŸüÇS½ã×û¿Ö¡o2ÿƒÿ~½?Ñ öõ?¯^ÔK§øPCâ«å5ÿ¦é¿ÖÿÒÈô¬dÿ§Ýgžìñ‚>ïo¯ÓŸOEù|ý~}-`‚·?_ÞKÿm“ì»é°ë/õoŒô«¼óÿO2{wO¯RÃV·š¿õ¿õÛaÓÕy9ÍäòóýWÈK ”COLûÉèO§>ô7vßwqRþ;r»zsK¿—õ{ S1NΟžp:Ú·ËÓÝ3Þ—õ ¿åן³^_c·Ÿõ¸ÙÿäÜ>Ƹ9ÿ¦iÎ:kGg®¶íú鯛ù…FÕ)mfžÝ¢ž´]z\ÿÔþòlœ.£=»åe}WT¼U ñnð¬(Kt Ì…Â}ï,£œ ´¾|¿?ë§Ýr%ñÓõŸßÈý;þuE&Λ®Å¶2ahnåùp±Go9Š8£8"XRÜ#Z0­(2®žwüþ~]¾}ÿŸœjiÓKJëml­óë{1#íñÿË•Áã¾'¶çø¿¯CÑNÕ~¿ÖßxßÇðÏó‡õ¿Þ2F ~Ù³É}¸~>ÐøQÉ}äžÀÆ[9imº­|ŸK|»}Âÿ—ü ÿJ=~{t¿½Í]øvÃPšåmÖËìåíd–ÒæÒ;« do´ï•W1ÝXÝÁ¹šÞKKˆ<©f·c±è½_éýÃþ$öø)¦»¯~Ý¿]¶WLÇ„ë:]Üð3ïHµhn}båä‡ì~DQK –°»š0¶°ù÷Úœ)(ŽXÐÜ\È„Êvþºþ?‡n—‘Š®¿jún+òo»BÖ™â CTšîÎ{Qák¦»•$‡W^IpÆ#Ùæb¶š <³#I0.·Q"%ZÖóÒÞÿþÙ ;énß~¦ö’Ïn·6ÑY¬fßT1Ë N³¸–k=_2„É"<.ìGê²1ý[[_qR¿+¾Ÿ½«÷s½~×nß}’Ž¶™,RÙE$N MÈtžBr¬ªÊØ •q¼dd ŠrÝú¿ë§õßp§ðGþÞòûrò[m×Ôl?ò @çÇ’Oý1ob¿ל¡|Kׯ¨ ÿs?åÕº&›èºLYNt§Ç}?¯'­·o^}>¢Ž¿?ëM?/¸U?>Ÿº{.ª‡õq×|Ù>N>X³è>h¸êr3þ{°·^¨uoì¤Öü°ßMÜSé¦þ^½dëÐM¸ö¹µ?€»‹üû~/ÑþZ~%TÑyóÓZ×Ȯݯü½úXšaß2b o<»1UO'sofΠ ÌNT’1€ªöùèSWÑ÷Oæ¶#šÓüaáMfö+K fÖês%³X80Úê¦÷G}f 4+ÙáŽÇÄ6Œ^ýæÑn¯á‚Ø<“ºiíýÃ[ß`¶ÞNëî·äßôëŸxVÎùì.µ»8f‹íÂæc綟§É¦Ëc 宧«$M¥iwÑK©éè–…íµíÁº„Û[J,€Þ]BÀß¾–·–§SKT¿}8OÛRÆY䵎ñ­·E´—PMl“”XÚxäIujÏ·×4 „Òum_Kž˜­t;¸oíä·Öäû5æ¡:T‹)ŠúCgm}x©lÒ»[[\Mƒ.ê_×áý!Y^ýmkùm.ôû›I,î­®.l®ã²Õ"‚á&’ÊøYÛ]Çiw»}šçì7v·B6;[]A>Ð’£3ü¿¯^ý¾ûI6í«µüíýWbÛ­¿õûXÇþ[ý;{þ\S—Oð¯ë§õ½írañUÿÿÓtÿ¯ø`³?ñ÷Ž§P¹éþücßžxïÞ—oOÉ¿ë¯éµþ7ù-BÓp…ö“ys÷·`)º“qùy«µJ‹×÷’Ú÷_{ûÆØŸôû þ[äíÇn¼ryã47­îûùþÒ&’µ8ú?Íú^ßð÷Õ¹þÌ©þÏÈçì§9Ç=½úýÑŒ±×Îþ·þ¿¯áÜ?Ç—üÿ­DŸþAMÿ^‰ŸÂ5ãœöÿ3š¨ü6*–öRßàùü+Ñz~OcÿÕþòñRmCéq¹ÇW‘æ¿>Ì ¾È¢Àó¶0À`S[?ë¯ü?Á¸Ë^ô=eó÷äí§™wUµûLÖ…K¤Ð¥ä´gkn_!ŒdÿÏ7(Ó¤†$ FbÚ={_/ëf&¿yMëµ]oýÕm7z¿ëR¶xe½X¤$”³Ò]¬ŠÑµÅ¶ó åîXʷκ)H¸¥ý_õa·ïÁr£û¥Oüû÷j÷f±ÀÔy scžÀÛríËr9ƒԩàgú?Uú‰ëöŸöóþºmçiUX_^Ë´2´6M¾6Ý„‹Œ<±dG>áò–ÎöO»*¶ÆS¢õ§ùÒ¤Þ¿ ;?NoÉ»à IÖu½V&x¼É™­‰6ëòH6\ VÄ‘˜Ë¿–¬å eý_×êøªÿŠ…(߯ž¿+ò襜š¤«|¶R h^éÛ솯tÙHàÌ-gÜbbòK#=”öÞt¬¯?›åF¾žkõ×ü8ⵟKÏ®šòÇä»ö×Í™v¶×ºh24¯1_Kä‡GÔ¬"žiÊ]MÁWÔtèZ=ÑyS‰¡ó4lSsPÕ¿¯×úí°©hŸmUýóm.ŸŸ¥µ5´i·Eu,P½2]‹‹`TEy$7RùW0\uWš)Êe>i>Tr,oR nðM^Þõ¯þ)|·¾ÖôZ£KO™.4xf@À5ž_‡‰Õ^ A,. r(à2œR¦šø––Õo¯ù~_q0þvºù|:+é~Ûï½®Hä6’Ùëýž3ÿ€À~qŸþµ uꂦ”gÿ^¥¶Ÿbþ×m‰.Çú÷ÊGéÓtC?¦{{t…ºõþ»þ_yU?†ÿÁý·Ó­ŸùìIy³ŸO>Û¿ý=Åôïþô¦·}>/–¨Tøíúúr#®ßÊ´¹ºÇåÛÜÉæ3**l†WÞÌûQU0™ÈQŒ±À&¤³÷?‡|,Ú¤ñ¾§u¡iM!’[c%ÅùÒbžôÛù'Ê‘Î&–_±n‹ÊV~à@>©ã²ø•ôé4ÖÓaÖuHEÛj&«m4þ†ÆÚÂØZÉ¥Ig±µXõÔïmõ+YÄQAÛ‰P ÍqüMÿ $CC¹‰ ³µÐ¯®¬îÁmïl?á#½µ×àw”›yF)½³œlˆÞéÖÖåöÍr(¶‰{«>‡ðɮóÛ¶¬æ]?}ôÚåÄÒ/”|©”Ý-Œïý¹‚ìp~Íæêtinä¾ñBÜHZ(|@±XŒÀ|»?ì _,yyuîK¹1s™þrB¬  ?¯ëoÏî4 É~ÀûHéÔÿ¢[sÁž{ôõª—ÙßáýzOËK(Äw©þ?ý²>›=>ZßTxÍßoôëž0?½?÷çéIôôÿ1Çí®Õ%ÿ¤Çü»ýÃíîŸóótyçþ^dúv~¼ã‡OwçV¦Ýy%~¶ûþò+/øòùdßúŸç§åš©|OÕ Ÿðáþÿ¥KúüA?ä¼ÿË?«ž¹Ï[Þ×_zÞºŠ?Â4þøü¿/¸lÇ:[sÿ.‰Î8ºLàô<óÏò¢/Þ»óoîMiKü Ïìï××oKØÿÖþð¼Asý—©èZÌóÉ„RÝi·ªJGm_Ǿ=Jîá˜,0Ù>s¹®w.KQý_ÓüoþÎïW+­üžßúZ]Ÿñ÷fÙoõwDcŒ‚‘÷Cu¯<çݾËói~¿Öß…¥/ø‘òŒíÿor'÷Y5çÕm,›ËV’ö¨¤hÔÛI;9f¶h,‘ƒ ñ/â5ÿNïÿ“¥çߥ»]ìhÅ‘ß]c¶2–že»òn I(&)@ÏîÕZWÚÊt^¯ôþ$Þï–šû¹·×ò¶[o•Ö·*÷šŒlÍ©5¾è¤uI6¼±8ÁÃ,¸ýÙRÙ`c?:0¡«[úëní=º%¿]Š¶ŸnÚßòî:u¿OÍÚéëªi6Óê-­ 3‹–•­RG¸WS [û>#æ3‹[ùrå!C†]¿¯¿ú_“‘Š§ný²¶»þ–ÄÚ>¹mr²G‹“(¹¹`Ékr‘ɘîfjd¬&eÙ"´aÝFúo’üéÛ^¯ÚÔéÚ_wã¯NUs>Ò湆ÆþÎæÝï“íiugjw,³·•qÝ4“C™R@M¼‰½Ñƒ+d¨÷Úß×븩üÓ¿þ—/5nÖíÚÊEx–ëXÒb“M¾›GÖÖÆݦURÖ×! nW‚áPL£Ë’Ý&ÏîÉxÜÏ¡»»õþ½?/¸šK÷4Õ·¦·i-£õ(-&¶¼Ó¡vF6’_íöòCi—"ûJ›’Åo ÒÒYâÂŽKìCgnÏ·é¯Ý[Š_îí+ÙÒ²ÛE(i}m¥õ×îû=êjú~§c9±¼Iš?-$ŒŽt"T{iBN€í%7'̹?- uëý_æ][*R¿HǧTãkvm®¯¯ÅökÂL[ý}±ëÿO1~žŸ¯JkâÓÎ×ùö·å÷_íñS½ºþò/Ïo+mÖ䲺¨Pá˜K"÷ À™>_˜Êã¨ÉÈà‚*¯ëú_’•¶’Wë%¿¯];íÒÉí,kmNÓ諒A{,··þD7W×W–:[ÛéòZ*è–2½¦ƒ –ªñÜŤEiÛ9kˆÝ‹nÎ+[ÉÙzÙ¿Ê/ú¹ªöv’E,2ÚÛÉ ÄžmÄ2A Å<»‘¼É£hÝ&“thÞd›œ²#܈Pñq¸òc ®<´óÚv‘a3mó +#´‹%<ÆgÚœ° › +»y4¶ŽH#[WH^ÊF°žÉnbžÐ˦ÝY´WmÔq´¢+«&‚x7îŠU4[z?¾þK·Ÿê?M³¶Óâ6¢qäEj%¸¹žK»ÛÇKdµŽæþú夼Ô/Œ±E=íä²ÜÌ#O2WÆæ?¯ëþîõqê”_þªÿƒ÷õåŒÖù2ß›þ>ÇCëkoÐ÷Ço«7ÓKiý?ŸæL/ÍW·:×Ö•?ø=¯ò¼‹Læï;¸¾¸ÁõôçíÓÐæ—õµ¿Ïïëå°âþ>žû¿ÝÁ´ÉIæâêäc×2cùçƒê9À§ýZÛðûµæTºßþ~Tßþ¾K~ÿ×fGg‘c,q󜃆“¡ùzý;ñ}þÿ×}W]>ð§ðGÓµºþ?‡n‰É©ŸìÅëÅ»[7°ÆO·ä@4uï¯ßÿ(»Qog÷Cçýt[ (fÓe†l׿ê—§=°:“íšk~ûüô~»úzÜ'¥)[¥?O³o-{ù÷»Gÿ×þ÷5;KkäŠÊò¸´º7×0JÇ,Y\$‘È‚ <çM+ßÉ__ë¿õŒÉÙÓÑ»ÏoûrÖ—v¿¬xk›g÷6‰©ÝI{§ØE#iÚ㬹™#›N×áˆGsG XˆõtAÓ™ËÇlH,t}î¿_ënþ8km'¦×Ñzí£kÑ+]³´ŽêÞK›{«i~ÑisäIï\Ín<Ï€¶DøWq3¢£µ?²ýWÏñÒÞší®¼­üqÿÿ8^þ{zôÙ•5M2+·šØÀ.!{æÈËöui~ÚÒ‹‹k•Rö·HìòÇ"å$“h;0®ªÚ_¥ì/ù{ÿpÿöýúþ_yÌÝ­í¥ïš‘]jºzØÚ[\@Zaª[fn$r°Àûª–É€¸ù\µ“þº|úK`†‘Ž›ÝÛþÞwZ6µþ¶nP[Ççi¶jR†Þg…™0W‰ò„´n$O(ÒAD¾'ÿþ_wMµÜTÿ… ÞÜzèß]<¼÷êŒyììõ-;ιµ³’ý4Æû9‘ÞKy•a&7‚älx¤VT3Ik ¸DG1º¥¼š·_Oóþ¶¶Ä5|;ïìoÿ’~/g¥ûùG2ëÃךdwHd[µ´¶VKÙžMVÍ£åíãÖ£ßÇ(óv´Ó­ÂpŒ"(Î-ÿË¿Nݧ±U5¥'{{‹]µÓ×úíª+Ê|E¡Db}FKÈEÄ ‘xÃ'Ú­À•5›öR Ìåâ+;‘Ý,Åt×tÿ®ßŸ­‚kÜZ·ïÓׯÇÑZüŸèt²xŠÏ}´:„7]À½„2ÜÆM³0•£>Uôy¶•ò#%ãgÚÄ P)_×oÇ·VãSו[z°·[;¿N—ë÷\ßšEØz~¿šÛ}”Þå-|ôVÚ6í¨CÿwG+_§n=OË4¿1ËÉÿ†Ÿþ’ý?­ïk‰oþ·P?ôò¿ú°ÿòTº…ÁüEŠ§ø×þ›¦:×ï]Óí“óÏ÷b÷Áúàzzî‘Çí¾õ%ÿ¤Çüû}÷ÏîMœÿÇÝÞ?ð!ñéÔôôéÎ3Mÿ_×ü?é•ÚíûÚ–öÿϯ—Þ2Ëþ<£ÿ®MŸûîNüööüé_Õ…OJq¾Öoñ­ÿ¦$_ò ŒæÇÿ׳cê{Sëþ_õ÷ûèÓö)ùü/_óµÂCÿÖçþ]ߺ¿”ã¿¥÷þº|½Vá?àK¿³O^ü«¥»ÿÃÚìÿÐþù§'Ì´8ÿ—’=1˜&çø¿ŸqÈÍ5×ÓõDKzãÿÛ'ÿ E:¬—V±Èªé$7¨é$a‘Ñ£ˆ::1`U‡ 8§ouú¯Éé·Kùy6_¼§uv•MoÝ%óÕ~>‡%an¾Ö?³YÉÒîí¯®,¼æ"|/ql‘ìbRÛÌÝ ºÛM–ÚJ]õòÝ[g­Ÿm.Áÿ?àŸþ•OüÎqz€Ößb,¥C9‰MÆ@ÈÒDK1R‡*­ÂˆÂ^_×õòûƒþ^vý×ßï¯ëåäJ#]݈-BÉQ$d­Ò†Fl˸°Ü¬¤õFÅ;h¼ïòü?»¨+ûIßùiÛÿ&õéoÒÖ±Í]èIs¨ßL‹5´Ò=®ÝKJ²ßÇ+Ú¬M%äY^B´£vãny#—(}=?WýÃ~*¿â‡ãN:?=ú}å;kZ°ûp»´_Áo1k«ý#m}ÑK|Ý;²„ÉjÍçºÊæ4 Š_×õÜPø«>Ž¯åNš{i¿o^¦îªi:Ìr=œÑË4WW-±Ñíï­Ÿó ÑÅqWmŽ@xZ@Ñî‘K rwmÿ_’ÛÓï%dïÿ?j|½þ«NÍ[Ë[Ý2õˆ¹†Ü ßi‰d¹<ô¾neÀÝòE.<²9oáwü¿/ŸõÛaÓ~ì|›ºã—É_þ©&žû´ËvÃm³µ—§ ŽFAN dR wëëøé»î&“½ z»û+ß^ÍÿÀì¾L¡wcÞ•>øeŠÙ.#¸¶"’e„e÷mu`ëÇ ‘tchÆ–¾Öý_geÿî÷ý½ÿ^£ÿ¤«þ½ôûãþx²]ÿfþß]72G·e½òE%ºHÙAòÛ½Ã@!}Ë,ŠìØXTÅ]«í÷_×qÖÒ”ÿçÎÖïÞþ~[š’\‰l÷`,Ë-¨ž ¼Sí̈ÊrØ]ÇiäH›dGeej­i.öõ¿õÚéª÷íúv÷>O——r®£¥ÛMqÆ ²µìnÉq5»ÈøBòZ̲ۻ(PxbÅò]ö‰^þ—û¿¯/ì9ýúû.ýv_;_¦Ö—>Ú%þšÐÇ¥ùv ÜyvÏh÷XF©Œ·z=Ä»ÑãméØÜ|Þlh°ªÃ¹…×}º~¾B—ÅJײž¾»ž–í÷y5¹j/ê6†àj6þ+bí=Ί~Ñ=°RJý«M”At¬c1;ˆRRŒd¹GT_™×õ÷¯_‘¿e®iZƒ­o­ä\ÆÖÌÞMÒ:™#ÚÏåÜ#¯•!*Éõ  K‘zücýéûéýpÐð)ôôŸ—Ë¿Ü/·ÿn+éý÷»¾¿v›ktÂ"Æêç·ÉmÛý™sì?_~H*tùùÿÃ~õü¼Ÿøiùô~Ÿð6Ö×oŸ6ÿÞèvÿ§KnÜ÷{û¦–Ñ[üþÿOžâ†²«å5óýÕ7ÿ¯é´'uØÏü¾OÛ¢ÇäLý7c4>Ë_É~ çòCŽõ?ëãü£×¨YŸ–nŸñùwÛ?òðØôëïú€ü¿!RÙïüZŸú_O?øYgì‘d¸üŽÿ‹Œãéùb‡»ÿ+~?®áOøpô/yþ{ÿÃŒˆĵ?ëȃ<ÂxÏz{{än:üúþ¢ðUÕ¿t·ÒÞç•öô×Îá/üƒ[þ¼×·ý2{Ëó§‰|ÿ&4¥.ÞÎßù*û¬ÿ˹ÿÑþùî9{Lvºüño?ùç¨-Qû_á¡ÞŸý|ÿÛ'é¾ßç°É?ãòÓ©ù.ñßþYÄ={›Ž£û¸!³ó_“ôÛþú°“÷é®êz³×Gº}Û[ÙuKõŽ‘á”A<¶·ª™-.âx~ÏvŠÙIÝßrÅ(h¤é"8!it'úwóóô@ÿ‰ýÊî•/^ÿMÌ}R¹þѺ°Ô£hïl­Yî9•Ñ£{Ù+«2êk ”U”a¤šÚW‘%ŽD1Rþ¿®¡ÿ//ÿNíéïÿ—_‘ÐÆY®nd‰–OÜZü’ç‡\œ‘÷[æN~`G*_§õý]§øiîOëþ’ qz`†·8Î|ÄvÝׂÜs‘Ÿ•¾žŸ«þ¿áÇŠ§¬?ô…òwߥ¶îåTØÚÝÜ\Kí>ñN³§EdöIª[=ȱÚ-o¥·%¾Ï¹c‚GŽ ìË€ªw•;IË9ßÅ¥÷ÿƒå÷üõÜ‹§‡i5ü–´ôí{í¿ÜjêUÍ”«Ímv +kt¨»ã2,x’mÑÍáe0u˜F ‘®Ðè–þº}ú[z­Í&¿w%o°“ù%é»óÝô±Jþ$¹W‰ãƱeoiåÜ81Çpt‘¬‹4E] {ˆ÷Ïh˜¬s•XŽÿåê¿?KŽ¦Ú<-ÿƒ#ÛþåróK!DXÉqí¯™ í¶E_2Eó#ž îÝ1©È À*þ¿¯ëò íúû ;½tÞÿžÛ¯µjiÔ½’¾aܨ) ÇÍäOò‰9GÏvrTƒòä•k®—Ó_óùiý0{ÃüoÿH–Ë«þº&µ·¸*eŒ\mŽePáö¬Ñ´rª3¨f@á$Ûó«‚B¢Ž?WðU†¤`–â mJâÙÐÁu¨¡U„*J»­õ«!Ø•KC䉃‚,2;±zçZ߇.ÇÙµ[˜­“OVÃÄ>©§4°E>Q¼KkºçO´mò÷Ú•¼Vë¹X±—pgÒÝ/¿¯ËËÎý·½?òñÿ×µo^yj½vÿ‡E˜pžtQH§O;ÿ]OŸ@_Äõïðç^}ü¾z¤VÒõ1ý¥u§^ÛýƒWkh'û*¸û%ç–&Yn4¹[ä’ÝØîع‹æûLJbvUý_ðßx“ýíE×’›ù.uúwûîmDyuÉt%­Ãm&9#cn¼¬ŠAVï¡ÿh€)¾žŸ«ßúüŠ‹÷ª+iͯ_qkÛ?K]‰n³,×ø1íQ€G@m  µÁÃŽ`“a1glª« ~Ÿõ×î[r«å;äzy;Ýo׺QKK˜sm‚4¶·pQ\±ë ƒ»”HÀÁRÔö—£~[|¿O—A?àµoùv·ÿ ק®›y«³ÿÓþú& =·½Çþј{óþxÆi¯Ñùôþ¿¤ÌçñRßZ?ëÝGù//ÒMpÍ·øÿ¾Ÿ¨úóïš_×õÿ ÷þ*_÷oð-=]>q·¼Ž?Ò¡ã³Ü÷õ{sÔç“ÏoÏúm¥×ü7OÏîþ$ÁSÿJ¥ÿÓæ)ÿÑëöOÓÏo¯¸ìCKòþ¿àÿxÿëßãοEý\ϽÓ-5Fž ¸˜ˆÅœöòÅ#ÃsksŸåÜÚ\&$¶¸%D‘œ•Ê>äb¬ú/W÷iëçÓ︒ýíM>Å/]§ç¥Ìcuy£Érº´·7¶Íw vÚÕ•™{«E{bÑ®±in¯v˜ühÖÓ;F.-­ÝÉqôôý_Ýý>¬qÒU=`öþâÝéù;®ºðÊë%ëy~t 8"HJ±EЇ,¹ýàSƒ˜‹7-¹h4=—§êÿ¯˜©üU­ÿ?÷?M÷Ûîl-î¡”‘ÄFîã+¨-!ÎÆÅ[†bÊ@9W-]ûÛòù~_qTövßÚÔÿÒ´]vwÿ± ¥³Ç›Yä„œ_Û \Hœ¬‘ªó¸C:ÎY†1Iîÿ¯óߧéöfÀ®¾Ôôÿ·åëùýöAe%ÂØÂÜ8ò3æA"ò|îŠM²#桘u]å±¹ —ð)ÿ×¥ù=¿¯ÍžÓ|²Þ\§³„•^7beËP‚Àãí5OYvÕoúÞ÷ûõó¹ó ÿrÿûŒ³sÿn@'÷iÏ=3ëÐúð[ëÉ¢?õ6½œ¯ÿ>—俯Ì}â úùÖüã'þ>béŒçü=0M(ïÓç·ÌU~4÷éô¿ü¼ŽÊúýÿ=n-ÎìC’Gú\'©þùþ~Ùõï•;ÿ_×õæ9ý4ö”õí¯÷Öÿ’ØmÊ!’Ô²«´¼ŠÙÿGŸ©#ž äôéž@£úüzÿ_ ¥½=¾?ý²{u½ößË«ŒþTd«вž )=Hm¹_BTŒãœ‘šE‰äBzÁ Ës G>çŽIàç$úô;@+ù›¦ÌœÛÆpaˆ€D¯ÈÊã#'±ô }<¯¿_éßüöDÛ÷ü _û~Jû|µåïÖÃc†ysû˜r!µÇîbÈÿ\ )ÇwÏO—£¢õ§õ·Üòòá§ø¦Ko‘5à:>íä[Å鎣n8Ïvþºÿ_ÕÅŠ¯nhýþÎ?ןÈ-rZï9ÿ¹0wùçùíŒ{‚Ëúþº~^}\ˆËÏúû%µ´å‡á×ÊâÛ²^?åêç¨ç~ÜñÉç àtéGõýzl:zú{IÇîŸË¾Öù±¶9±çûúI&qÓ?ŸåO¯gë¿å÷ŠŸðáèÿô¹ÿ/»ŸôfëΔÙõéõüð(ní¿?ëï¥>Ö§ø¨í×Óo¼ÉÓòÖ:¯Ê9»º×¥¨ä±þ,uR¦0i¿‰ôÕþçÿ#wJ6Wnœtèï£Ûmo·ÊçÿÔþùî2Ð srä<‰Ï¶î{÷"š_×ë³ròÒäJצßó»zòM~M÷_uâH3skÈÎ'Ç?ì)àñÐsŸ¨éF–/ø?§Ü9YNžÿòõmýÕ÷>ö¿m7é0ôÏ‘qÜc ö9çŒ3Ãeú¯×úÛ︟ñ#þ þtÀƒöÑÓ‹Oïùíÿ×õü'jü½ÿ¸ûõßñ¼Hóö›•â+SÔ¦~;ã§gžsGõý}ãK÷³ìáOÿJ«ú[úbDÚ/3†€ýñÞß9‡*¼r2 Úûz~¿×ôÇyTÿ>V§ﶭë³Ó©€ú±_]ê:-áÓîÉX$´vitk¤Fèg° ˜ÝŽÛÛ?&c43 P £Ù]®Ÿy1øªkö’ÿÊpß­úï³K[ZjÐA9‡\·Eë_ÏbG“M¾ÃLêÐê(‚gŠY,µÒhÓ|A&qï¿Eù/Ëb¡ð½~Ö­ÿð/éÿÀ±±f÷BtX.ày.?)ÂJщåd)¹Œ´€ü…$‚0;›wÊÞÿÃoýkµû+Øš?éïM}Õ$¯÷køõ$±Êb"“ìÅÌR‘ŠªÞ[9D„ u%‘°° ÒzýËðVòþ»î*À‚ê©ÙÛº¿õ§§Av—ÒÂœö=‚A±ïDZÎj¥¤¯Ù¯ë¯å÷‰¯öv—üøivþýtùé}Õ®%Õ¬olîQw˜£Ã+•a±QTŒaAUÎÞ=û|²·^¨¹YÓ–é{?ý·w¾}¼î-Í»%¼'Ÿ/,4³,åÒ"ÆÅ•J®{áO¯9&—åÜU~>zI_§ïa¯ËÒ^›ñ\¯’ÊÉ›˜‰3E]ùD-ïn ¼“´®6•§{~+ïûûù~l©»(¯ú{KðŸo[uûÅu¼V·ó%µùÃ*±¼9ù%Ûó§ÆAÊ6ìdlÏÊ.»íÓå¿—üKâ¥ÿ_?öÉÿ]ûkbÊ}¯?¼ŽÙG<¤îǶӃŽyÏ+Œg8¤XäY”í+A¤LìÄœ–ÈhýIÁÜÄŽ¸à(BnþÔÛcµ'ìéË\:ñæÉųþqœ¯§•ßéýÓ§?Ÿ"éý÷÷¿=,´êœ£S{ö»VdùVüµÄ£#÷»z[¸õÏ$'$Òùü¿^ßÞ%üIéöiþR駯ç}BoüÛ¼Ee“4yÍÌÃþ]âèÇ®9鎸 eëúÛúï¸Gâ©­ý轿¹/§äýt t½ÍÙ3ZÆMË’« Ê«û¸ºHgˆ¸ rLiƒÎ@¦úzw¿ü7 Cþ^oüYïé >Ÿð6ÖÉ’ZÇs²Oßß´O“äÒ¾?ÒxÜ2'‚s· Š…ºmÏ/;>gu׮ɽ/mD³‚_)Iº˜>ͱ3#ýÓånç=›§’Íî﮽¥ùýÄRþ~úS_§Ÿæ£`/ú&ÜŒzË23ß{ätÉÉÜžº÷þ¼¿/¸¦­MÆûS’Më´ZMÛ]íý\ÈÓý“Woü~Þc 2GÙ-FI=#8ç®}¶µk­ÿ®ßð|÷éN-ßH'kk¤o¢×úê·?ÿÕþù®ÏiŸùúÀÿ–òr;ŽÙõäS]}?®Þ¿çk-éÿÿ鹄˜ûM®AÈóúŸ»^?ýGó¥ý_×æÆþ:÷ñ‚ó×O»û×´UÆ.a?ôÆç¿ûPsÜò‡ž½*¾ËßF¿áÞ²[­úƒøãþ ÿéPþ¶û®lsöR:ÿÓo§¿zœb¤Kø¯þ½íèj¹Æså[rH㛎¼q×<ƒýôù¿Ó§õ/âKüÿÒª ˆ¤ÞõäÛƒÈç÷?î£òíŒåG²Ó¿Ïú½¿¤5ñTÖúÃÿM¯]º÷}­aÐãÌ»ÆGïê9ýÄX=ÿšþ84=—ÎÞŸð÷ôùŠ?Oñ+úòGñµ¿28cŠX®b–5š&º^9FO$tb !SÈØu ®@;…ºõ_™U.èËþ½ù-¢ºëÓ_òM »—¾è¦P²BwáYN&„ü»w99ã^Iê`п¯ëþÕn>IStâÿ§­·³µˆç¼¶ÄEäò±qúõx3óàíóU7`àœ”»…×Ñüí­¿/žÁShÿ×Ú_ú_mo·¼’K‹y¾ÌðÍ©ö¤]É*Ÿ*^­çÓýœåŽÿ×_ë·ãi9|Tßißÿ$ŸßýveÀñð¹çœ‘Ø.3ž: ãÐ nEÀïœ{öÇOº1뜟Nù p>ÐØa@=x‘úúõ¯ÚÉv¾‘·m{ë÷\[2|Àûóv_ùùš‚h| §ï*kÿqeÿ-°Ö‘‚£kC‚wr œ‚0sƒE9nýXR_¹§{ë{ùÊ]íÓÏÏFÑÉ\øGK’Ô,Lú6£%¹–{Í*O³}°ý™#Ù}mͽʆ$l¬Rì@‹:#:Ð÷~¿×oë¾â_Àíz;¯ú÷o˶þ¶-K6»aj<øÓS·Šynm#ó/$U@FûF猾Û6m¸T &H¡n½|ßÎÞë‡èT•éµ®°¶š;YmÒöïçÕ’Eâm+Q¶ì×P¼ÒýžO±¾ë-J0óBèeÒuíuŽdK 2g?0+H*|/ºqÿÒ£¶Ýïù^Ö6./"_%\´ln¡SæÄêª|Î&Óy»Xà$MuÖÚ?ëæíëòô·_ÞÓ×þßß®¾Žýu°Ùg´¹’Ü$–·npV†Rª©0$Œ7ˆö=X ¯§ê¿¯Ã¨Kzk¼Úô÷&î¼ô{ô~¥¯³ÛbÞAÈdŠ0An¬®ˆ6ãï~q†E -âHçÜçŒz30öÈõ mŒE ÚhÌç æ8Àùþ_÷”«~B€êõé·õùy\jÅÚ'!G)'sãø8ÝŽÜ`ŒûàÑåý_/¸…­IéÒŸ^ÜÏmz뒲Е`‰^S±NæSóüì€à€Iÿw×''iý[þ_yJ+]/w~ÿðÝöûˆ¢‚{’ÐÂKNù&O\_x”<3‚yï’ 7ÓÓúïývØPÞõñéÛHÿ_ðÄC^c$P¡iQ¡*$ʆ* ¶3Ð’=¤To}^¼ÏU}›Û®«gmü¯am0GÇÍ×ó+zàvúžF1Mîÿ¯ëñ·w¹Õ¡oñt·Û’îö·ü>®J0ðm8ÃDT•^¡‘ÁÇ\N>ñìsUþOòù|µûŒO*Åá}6$U –#U $PJª¡¸Îçc‚ ×õýz™ÞÔ\­{Rr·~X__[–èÿ×þú'Æëúø¡Ïú©zŸ¥5ù^}ÿá·3ŸÅKþ¾uó„þï_ò ÍÅ©ôó¸Çý32Gò?…"ŸÇKþâÿé ×ú}7ÿ×ÅëäÎg˜}rO㎸àšwÓæŸçýà ÿ?àŸþ•Oúÿ‡ÿÇàÿ¯VÿÑ«GÙ~«õþµ¿ëþ^ÿÜÿ§#ýÃŽOøø˜±No_nßeÛ_ÏîþµèL‹S·%?Χü(”‹‹Îz‹nÇ´d›³ø v;¿…½—«ë~ÞJßÓÒéFâýú¾N—Ï÷wò¶ï¯Í}™"=ϼ ÷ÀýÌcù ÿú…&´O½ÿý›ÑEGâšó_ŒWùwû„·çíõôÿÍsøuíÆ;Ðúz~¬!¯?ý|šóé¾ÿzZÛ®‚ÚãÉéÿ-&×àú{úúô4åñ?ë PøWøçÿ§%ýÃŒ²ÇØá#§’>ÿÎ;~4žï×ðÿ† Ç£ÿÒä0ø—¸ûè†üÿ×߯ÞJ_¸Kþœ¿=©é~ÿw¯5ì>_øõ'òÅNyþì|äàñíýÞø…ºõ*ÖŸòﯜw~]} º®Ÿc}lëygmr¢ÜA­>,|ì»Æ6©RªzªŠå¯Ý¨U·#¾¾ô>þxÛ¶Ï×˹{¡\ÁM#[Ôtö{».;­š½”_?w9I ž§W¶Ifúz~¯úéù9(_÷‰¿ùy.–û1vµº+^^«rA ÎI‹·s }’2Cg·y£¡É ´Ðœ Ò)y¾¯W¢µôZ_e¥úï¦Ãtô˜C’a"âN)šqʳŒÏœàmowêȧð/Y[ç—?ëTVÕ広E¾šÈ§Ú¢±–H7’ª]$dAd,¨AʾΤUýÖúò¶½m¦ÖëÛî{máH4È¢ŒHìÀEleG“¼#… 9<Ó{ín˯–ÝH–”&÷µ½ofÕ7¿[6µî½OÿÐþú&8k{€?ò¦‚'½?úùÿ¶Oñþº1$ÇÚ-Ž;Íøf0ÃÓõ!´~©~×ü8ßÇKþâÿél‡ý*Û¶aºïÐþàœðO¦>¼à:?Uù0k÷‘ÜŸÏÞ‡£éÛÖÖ°§þ?—ÞÑÏÓGþ>¿ªý­þòå÷ýÁúqÃHrÜMèc€š`,N¼ö,=—§êÁ_ÚÏO±O^úÔÿI»êqäè³Ó±Ç_ÿY£¢ùÿÃþŸ!Çø•µßÙþÕ½7ëê:/õ—_õ×ôòbú?¦1Céé~Ý^Ö>*¾Uù{*OólKn>Ñÿ_Rñþö܇ú{ö4=—§êÇOy÷ö²Ó¶ÝöùúõÈþã¦?{?öõ?õëÓ>ƒ¡îúyvü¿/ºâ ½Åÿ_*éÙ~wcÿpóŸÝæÜþ>¿ røŸËò5û¸i§+ÓþÞ—õÓÑl7ìãèlÛÿÓqŽ=ÿ,Rëý}ûuÞÖ}µ%·î^ݳz/–ƒç#ì×ýJÿ$9ÆO¦$Ñתþºÿ]·*—ofïn‹—~¿==/a÷êûðÿèø¾œú^”¿¯ëë¶áSà~°ÿÒãÿòóAq‚"ïþ“àCçÿ­Æ:sÓ ×_OÕ «kÙ÷ué+íÿ/çÖé|®Ô‹½mÿ_ ÿ KÛÿÕŸÀ_Kí濯øpŸÇC_ù|´ïîTü¾_’”Äí*·ÌpJœ*àgæ9¢áN[²äšF†-ׇ4{‹™/EœVºŒÐ=³jvJ¶—þ[ì ðæ®A—~eåX­gÇk¨é¿”ƒX¶òcÉT²Õ–%u ÒȲ-®¥ ùß “Ê«-O§Ï·ëú%ëk"töõöi|ùÛ¿}´Ýzkyh[_Ù5ù€Þ´WwFöÚmç—ozRÜJe0ÛHâUU;¤ær nÜ 3*þ¿¯¸Kø“ÒÞí=u×Z›koëìí+j.Ä×&9 eóPÞ'‹?¹'ÍW•—hço•‡#–LüÀGâ©þ(ÿéýžíMÑ1C Y\…ŽfYM…Eó¢X‚“½Ü`Ÿ¸~Z§ö}?¯ëüØGz»éV_úL:õ}ôûî„·žs¸}‰”™7´Û¤Ë‡à“%¾\—Û…‘KªÓ·’~»¯/ÅÚí6w·óÎÿâNÒí­÷óï£,Ú ¶ñƒœüÜzfGçŸ×éCÝú±Sw§ºóésZíÛôÖ×2u‡1ÙYÀãϸhç–ûNŸ¢j¡åKª\E#)Ûº%m­Î)-tïýy~qW÷oåý»ù|º€âÇéiŒúþç®?Å>¶Ö÷²ïäe+¼4Þ·xi¿;º/õgÿÑþù®ÓÞèN°ÍõôÏAëžM5×Óõ^^}ï§KûÑ?Š—ý}_úLÿ­þñ$ÇÚm01þ¿ÛþY¯8ÿ?­ð5íý|ÿX·ñÒÿ¸¿úBþ¾ëôNní¸?ê®ûŸ[\žøÀüŒQÑú¯É…ýø¯îOó‡ùÿVÿÇòz}’OÏ΋ü?Ý ªýIÿ—ß÷ÿnŸñó8íå[ž;|Òöü:ö=–õï¯è5üIÿ‚Ÿþ•WúÛïn.ø}Ÿ«œÆzàŒútížrhè¼úÿVµš·ù­E{ZºÿϾ–ûý? Rºrt8ó.Îâaü]?qL0 u÷êy=£éúþºú|ÇiTÿ_ßN¥µÖþV°–ÝnAQ‘tÿÅþî:7ëÎ}0y4>žŸ«þ¿á…MßÚuµY«ýßÖò¾÷ 1ˆ>êÿ®¸þ/úz›ÏׯN3CÝúÿ[Ûúï¸PøWý|©çÿ/g¶ÛWÜmˆÿC„m_õXûÜunƒ?×Óžôåñ?ë§Ëòû‡OøpÖú>–ÚO½¯ëo¸güÃ:)ÿB<ƒ“Ÿ ôò÷íÖ•ýëùßñùþx£oaåì_þ›bÍÅ“qǾÿ‡Ž=zcŸ™hþ¿­¿ó_d©üÛþ}ö}—Dõ×Î7ù¸»âÝ'÷‘qÏü÷ú߇Z–ÿõòô±U¹/X~!éýwØ[ƒþ>!œ£>ÜžqÏ1Ö…×ÓõD×û:Ëê_/Þ/ëU÷Y‰q‚ÖßõóÿÇ%Çåôüø¡lý?_éïÓi}—?Š–ÿÅV·~I}ÿ…·ò•œ8üóŸ^¼úúûóH°ÿéþ}?L0ùyÁy2Oü¶ëÓüàŸC•ôò»ýöŸøŸööþ}­çs#UÐt_¶Ö´»=J8Å´°‹¨CKo"<¬’[Ü..-äR[ ‘Ÿ™ÆHw¿¯ëþî"?Ä©þvÿÉÆ›MFÕ¥:]Ä/R@>Á¨+Œ1Ä¿¹‚î/ßÛ?•µ#–ax©±‹æwVö^Ÿ«õþ»^Ã÷ª«»sÆËþáÇúù•¢ñµ¥äÖz´3éÓM;˜î%ŠGÒä‘`·/ Z’Gä B¶ý“ˆ“yqÕGÓÓúû¶éø^N;Ô}êËÿIŽ½l»kß} ë7Y#’HÊÉ—²I”ÉŒ‰”aÁåI“Ö­§Ÿ¯ãÿÈþ¢†Óÿ¯µß'¹«jÓ}´¼¡kË[ =¯ogŽÒÚ‘¥¸™‚FŠ$|àÿðvF€Èí„Œ;´=ß]zõáCþßÿÓ“34ieÔam^Hd† íU4Èçf6F=ëxÐ2†³’óy&Îò@<Â)A…ºõ-+ÅwqüZz/óésdœØ×y#±ÿGÏNÃü=÷­çm=zmùýÆRÿvw×ý›[kÜÙ®›ëú'±ÿÒþùç5¯ý|ƒÏýq˜_æU®·è¿à~¾ãø©yÔK{o úÿ]·Aþ‘kÜæÖ>ÞÜR |t½jé üÂAþ•nrxŽä~~GøgëM|/Õ~ õ«ÿNê¿[:~½ü¶³N÷ÿÉÿ^’ü÷ƒ§¿È×Ã/—çýLrÞãÿÛe–›yßN[XéÉãüóÿëÇ^¼T”.?žh~Ñúb{ÿ¬éÀŸ\Ó蟛ü-þ}¾ñ}«qKæäão¹^ÿ+?ˆjô™ù?ê­Áü Ý:õÉÏØði¿‰?ðÓüçú ýíÏ8ËÇÿ¢S íõÏ~ئúz~¯úÿ‡7«Ûž:Ü8ëý?U-9bå` ½Ó†V•ÔÁ•t`U`A™;GÓÉ_ˆGâ©ÿ_䌼+£‰ÞæÖôÙÖæR_J¼¹ÓÖE ›RHacP±ª€‘&ѸÜÆùWó~ŸŸÝvÞõò˜í;Ã:LrE¨J—Wתó¸ÔïnuMҾЫpæ !çÈùÉÞåf-CѵÙÿ]ÿ?¼T¿‡—óöþy/OëÈÞˆ¡/ý{;«'>¹üqÛš/­üïýoù}á )­ÿ‡ë¼¯Ód0Œi­ƒÈ±o^ÖÞœö3­wêÿ­R%¦ÿØ?þâüÿáúŸÿÙcanu-1.3/src/000077500000000000000000000000001272335422300130755ustar00rootroot00000000000000canu-1.3/src/AS_RUN/000077500000000000000000000000001272335422300141245ustar00rootroot00000000000000canu-1.3/src/AS_RUN/fragmentDepth.C000066400000000000000000000306721272335422300170300ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2007-MAR-28 to 2013-AUG-01 * are Copyright 2007-2008,2010,2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2007-SEP-04 to 2009-AUG-14 * are Copyright 2007-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-NOV-21 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "AS_UTL_fasta.H" // A quick hack to compute a histogram of coverage depth using // the runCA-OBT posmap files. #define HISTMAX (8192) #define DEPTHSIZE (128 * 1024 * 1024) #define FRAGMAX (1024 * 1024) #define MODE_HISTOGRAM 0 #define MODE_SCAFFOLD 1 #define MODE_DEPTH 2 typedef struct { uint32 lo; uint32 hi; uint32 de; } intDep; static int intDep_sort(const void *a, const void *b) { intDep *A = (intDep *)a; intDep *B = (intDep *)b; if (A->lo < B->lo) return(-1); if (A->lo > B->lo) return(1); return(0); } void computeStuff(uint32 *V, uint32 N, uint32 B, uint32 E, uint32 *mode, double *mean, uint32 *median) { uint32 histogramMax = 128 * 1024; uint32 *histogram = (uint32 *)safe_calloc(histogramMax, sizeof(uint32)); uint32 histogramBig = 0; uint32 meanCount = 0; uint32 i; if (E > N) E = N; *mean = 0; for (i=B; i 0) { *mean += V[i]; meanCount++; } if (V[i] < histogramMax) histogram[V[i]]++; else histogramBig++; } if (histogramBig) { fprintf(stderr, "histogramBig: "F_U32"\n", histogramBig); exit(1); } // Find the mode -- except for 0. // *mode = 1; for (i=1; i= histogram[i]) { meanCount -= histogram[i]; } else { *median = i; break; } safe_free(histogram); } void outputResult(AS_IID lastiid, intDep *id, uint32 idlen, int mode, uint32 *histogram, uint32 *histmax, int stepSize) { uint32 i = 0; uint32 lastpos = 0; switch (mode) { case MODE_HISTOGRAM: // Update the histogram // for (i=0; i 0) { histogram[0] += id[i].lo - lastpos; } histogram[id[i].de] += id[i].hi - id[i].lo; if ((*histmax) < id[i].de) (*histmax) = id[i].de; } lastpos = id[i].hi; } break; case MODE_SCAFFOLD: // Report mode, mean and median for this scaffold // { uint32 N = id[idlen-1].hi; uint32 *V = (uint32 *)safe_calloc(N, sizeof(uint32)); uint32 mode = 0; double mean = 0.0; uint32 median = 0; uint32 currStep = 0; for (i=0; i N) { E = N; } computeStuff(V, N, i, E, &mode, &mean, &median); fprintf(stdout, "%s\t"F_U32"\t"F_U32"\t"F_U32"\t%f\t"F_U32"\n", lastiid, i, E, mode, mean, median); } safe_free(V); } break; case MODE_DEPTH: { char *seq = (char *)safe_malloc((id[idlen-1].hi + 1) * sizeof(char)); uint32 j; memset(seq, '0', id[idlen-1].hi); for (i=0; i%d\n", lastiid); } break; } } void processScaffold(AS_IID lastiid, intDep *in, uint32 inlen, int mode, uint32 *histogram, uint32 *histmax, int stepSize) { uint32 i = 0; uint32 idlen = 0; intDep *id = NULL; // Convert the list of overlapping intervals into a list // of non-intersecting intervals annotated with depth uint32 islen = inlen * 2; intDep *is = (intDep *)safe_malloc(sizeof(intDep) * islen); for (i=0; i' letter.\n"); fprintf(stderr, "Depth 0 through 9 is encoded as '0' through '9'. Depth 10 through 68 is\n"); fprintf(stderr, "encoded as A-Z[\\]^_`a-z{|}, and depth more than 68 is encoded as ~. Decode as:\n"); fprintf(stderr, " depth = letter - '0';\n"); fprintf(stderr, " if (depth > 9)\n"); fprintf(stderr, " depth -= 7;\n"); fprintf(stderr, "\n"); fprintf(stderr, "!!WARNING -- The input frgscf MUST be sorted by scaffold ID -- WARNING!!\n"); fprintf(stderr, "\n"); exit(1); } uint32 inlen = 0; uint32 inmax = 4194304; // 4 million fragments per scaffold should be enough (but we'll realloc later if not) intDep *in = (intDep *)safe_malloc(sizeof(intDep) * inmax); char line[1024] = {0}; char *cont = NULL; if (mode == MODE_SCAFFOLD) fprintf(stdout, "iid\tstart\tend\tmode\tmean\tmedian\n"); while (fgets(line, 1024, stdin) != NULL) { AS_IID iidjunk = strtol(cont, &cont, 10); // read id AS_IID iid = strtol(cont, &cont, 10); // scaffold id int32 beg = strtol(cont, &cont, 10); int32 end = strtol(cont, &cont, 10); if (lastiid == NO_IID) lastiid = iid; // Did we switch to a new scaffold? Process this set of intervals. // if ((iid != lastiid) && (inlen > 0)) { // This scaffold is the correct size // if ((minSize <= lastend) && (lastend <= maxSize)) { processScaffold(lastiid, in, inlen, mode, histogram, &histmax, stepSize); } // Setup for the next scaffold // inlen = 0; lastiid = iid; lastend = 0; } // got a new scaffold // Save this fragment. // in[inlen].lo = beg; in[inlen].hi = end; inlen++; if (inlen >= inmax) { inmax *= 2; in = (intDep *)safe_realloc(in, inmax * sizeof(intDep)); } if (lastend < end) lastend = end; } // process last scaffold if ((lastiid != NO_IID) && (inlen > 0)) { // This scaffold is the correct size // if ((minSize <= lastend) && (lastend <= maxSize)) { processScaffold(lastiid, in, inlen, mode, histogram, &histmax, stepSize); } } if (mode == MODE_HISTOGRAM) for (i=0; i<=histmax; i++) fprintf(stdout, "%d\t%d\n", i, histogram[i]); safe_free(in); exit(0); } canu-1.3/src/AS_RUN/replaceIIDwithName-overlapDump.pl000066400000000000000000000035641272335422300224230ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz on 2013-SEP-22 # are Copyright 2013 J. Craig Venter Institute, and # are subject to the GNU General Public License version 2 # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my %IIDtoNAME; my $fastqUIDmap = shift @ARGV; print STDERR "Loading UID map from '$fastqUIDmap'.\n"; open (F, "< $fastqUIDmap") or die "Failed to open '$fastqUIDmap'\n"; while () { my @v = split '\s+', $_; if (scalar(@v) == 3) { $IIDtoNAME{$v[1]} = $v[2]; } elsif (scalar(@v) == 6) { $IIDtoNAME{$v[1]} = $v[2]; $IIDtoNAME{$v[4]} = $v[5]; } else { die "unknown format '$_'\n"; } } close(F); while () { $_ =~ s/^\s+//; $_ =~ s/\s+$//; my @v = split '\s+', $_; die "Didn't find IID '$v[0]' in overlap '$_'.\n" if (!exists($IIDtoNAME{$v[0]})); die "Didn't find IID '$v[0]' in overlap '$_'.\n" if (!exists($IIDtoNAME{$v[1]})); $v[0] = $IIDtoNAME{$v[0]}; $v[1] = $IIDtoNAME{$v[1]}; print join("\t", @v), "\n"; } canu-1.3/src/AS_RUN/replaceUIDwithName-fastq.pl000066400000000000000000000073061272335422300212550ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/AS_RUN/replaceUIDwithName.pl # # Modifications by: # # Brian P. Walenz from 2012-DEC-13 to 2013-AUG-23 # are Copyright 2012-2013 J. Craig Venter Institute, and # are subject to the GNU General Public License version 2 # # Brian P. Walenz on 2014-OCT-01 # are Copyright 2014 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Usage # fastqUIDmap fastq fastq fastq my $fastqUIDmap = shift @ARGV; my $inFile; my $otFile; my %UIDtoNAME; my $namesLoaded = 0; my $readsRenamed = 0; sub loadMoreNames { my $lim = 100000; undef %UIDtoNAME; while () { chomp; my @v = split '\s+', $_; if (scalar(@v) == 3) { $UIDtoNAME{$v[0]} = $v[2]; $namesLoaded++; } elsif (scalar(@v) == 6) { $UIDtoNAME{$v[0]} = $v[2]; $UIDtoNAME{$v[3]} = $v[5]; $namesLoaded++; $namesLoaded++; } else { die "unknown format '$_'\n"; } if (--$lim == 0) { return; } } } while (scalar(@ARGV)) { $inFile = shift @ARGV; $otFile = $inFile; if ($inFile =~ m/(.*).fastq/) { $otFile = "$1.fastq.RENAMING"; } else { die "Failed to generate output file name.\n"; } open(F, "< $inFile") or die "Failed to open '$inFile' for reading\n"; open(O, "> $otFile") or die "Failed to open '$otFile' for writing\n"; open(N, "< $fastqUIDmap") or die "Failed to open '$fastqUIDmap'\n"; $namesLoaded = 0; $readsRenamed = 0; print STDERR "Renaming '$inFile' to '$otFile'.\n"; while (!eof(F)) { my $a = ; chomp $a; my $b = ; my $c = ; my $d = ; if ($a =~ m/\@(\w+),\w+\s*/) { # UID,IID $a = $1; } elsif ($a =~ m/\@(\w+)\s*/) { # UID $a = $1; } else { die "Nope '$a'\n"; } while (!exists($UIDtoNAME{$a})) { loadMoreNames(); #if ((!exists($UIDtoNAME{$a})) && ($readsRenamed > 0)) { # print STDERR "WARNING: Looping to load more names; out of sync?\n"; #} } die "Didn't find UID '$a'\n" if (!exists($UIDtoNAME{$a})); $a = "\@$UIDtoNAME{$a}\n"; print O "$a$b$c$d"; $readsRenamed++; if (($readsRenamed % 10000) == 0) { print STDERR "Renamed $readsRenamed reads using $namesLoaded names.\r"; } } print STDERR "Renamed $readsRenamed reads using $namesLoaded names.\n"; close(F); close(O); close(N); if ($inFile =~ m/(.*).fastq/) { rename "$1.fastq", "$1.CA_UIDs.fastq"; rename "$1.fastq.RENAMING", "$1.fastq"; } } canu-1.3/src/AS_RUN/replaceUIDwithName-posmap.pl000066400000000000000000000046631272335422300214410ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/AS_RUN/replaceUIDwithName-simple.pl # # Modifications by: # # Brian P. Walenz on 2013-SEP-22 # are Copyright 2013 J. Craig Venter Institute, and # are subject to the GNU General Public License version 2 # # Brian P. Walenz on 2014-OCT-01 # are Copyright 2014 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my %UIDtoNAME; # Usage # fastqUIDmap posmnap posmap posmap my $fastqUIDmap = shift @ARGV; print STDERR "Loading UID map from '$fastqUIDmap'.\n"; open (F, "< $fastqUIDmap") or die "Failed to open '$fastqUIDmap'\n"; while () { chomp; my @v = split '\s+', $_; if (scalar(@v) == 3) { $UIDtoNAME{$v[0]} = $v[2]; } elsif (scalar(@v) == 6) { $UIDtoNAME{$v[0]} = $v[2]; $UIDtoNAME{$v[3]} = $v[5]; } else { die "unknown format '$_'\n"; } } close(F); my $inFile; my $otFile; while (scalar(@ARGV)) { $inFile = shift @ARGV; print STDERR "Renaming '$inFile' to '$inFile.UID'.\n"; rename "$inFile", "$inFile.UID"; open(F, "< $inFile.UID") or die "Failed to open '$inFile.UID' for reading\n"; open(O, "> $inFile") or die "Failed to open '$inFile' for writing\n"; while (!eof(F)) { my $a = ; my @a = split '\s+', $a; foreach my $a (@a) { if (exists($UIDtoNAME{$a})) { $a = $UIDtoNAME{$a}; } } print O join "\t", @a; print O "\n"; } close(F); close(O); } canu-1.3/src/AS_UTL/000077500000000000000000000000001272335422300141245ustar00rootroot00000000000000canu-1.3/src/AS_UTL/AS_UTL_alloc.C000066400000000000000000000024051272335422300164320ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2005-AUG-24 to 2013-AUG-01 * are Copyright 2005-2008,2011-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Eli Venter on 2007-AUG-30 * are Copyright 2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz beginning on 2015-OCT-08 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ // Nothing here. Left for future use. canu-1.3/src/AS_UTL/AS_UTL_alloc.H000066400000000000000000000071031272335422300164370ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2005-AUG-24 to 2013-AUG-01 * are Copyright 2005-2008,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-AUG-11 to 2015-JAN-31 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-08 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef AS_UTL_ALLOC_H #define AS_UTL_ALLOC_H template void duplicateArray(TT*& to, LL &toLen, LL &toMax, TT *fr, LL frLen, LL UNUSED(frMax), bool forceAlloc=false) { if ((toMax < frLen) || (forceAlloc)) { delete [] to; toLen = frLen; toMax = toLen; to = new TT [toMax]; } memcpy(to, fr, sizeof(TT) * toLen); } const uint32 resizeArray_doNothing = 0x00; const uint32 resizeArray_copyData = 0x01; const uint32 resizeArray_clearNew = 0x02; template void resizeArray(TT*& array, uint64 arrayLen, LL &arrayMax, uint64 newMax, uint32 op=resizeArray_copyData) { if (newMax <= arrayMax) return; arrayMax = newMax; TT *copy = new TT [arrayMax]; if (op & resizeArray_copyData) memcpy(copy, array, sizeof(TT) * arrayLen); delete [] array; array = copy; if (op & resizeArray_clearNew) memset(array + arrayLen, 0, sizeof(TT) * (newMax - arrayLen)); } template void resizeArrayPair(T1*& array1, T2*& array2, uint64 arrayLen, LL &arrayMax, LL newMax, uint32 op=resizeArray_copyData) { if (newMax <= arrayMax) return; arrayMax = newMax; T1 *copy1 = new T1 [arrayMax]; T2 *copy2 = new T2 [arrayMax]; if (op & resizeArray_copyData) { memcpy(copy1, array1, sizeof(T1) * arrayLen); memcpy(copy2, array2, sizeof(T2) * arrayLen); } delete [] array1; delete [] array2; array1 = copy1; array2 = copy2; if (op & resizeArray_clearNew) { memset(array1 + arrayLen, 0, sizeof(T1) * (newMax - arrayLen)); memset(array2 + arrayLen, 0, sizeof(T2) * (newMax - arrayLen)); } } template void increaseArray(TT*& array, uint64 arrayLen, LL &arrayMax, uint64 increment) { if (arrayLen + increment <= arrayMax) return; LL newMax = arrayMax; while (newMax < arrayLen + increment) newMax *= 2; resizeArray(array, arrayLen, arrayMax, newMax, resizeArray_copyData); } template void increaseArrayPair(T1*& array1, T2*& array2, uint64 arrayLen, LL &arrayMax, uint64 increment) { if (arrayLen + increment <= arrayMax) return; LL newMax = arrayMax; while (newMax < arrayLen + increment) newMax *= 2; resizeArrayPair(array1, array2, arrayLen, arrayMax, newMax, resizeArray_copyData); } #endif // AS_UTL_ALLOC_H canu-1.3/src/AS_UTL/AS_UTL_decodeRange.C000066400000000000000000000074241272335422300175460ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2012-FEB-12 to 2013-OCT-11 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2015-MAY-28 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_UTL_decodeRange.H" void AS_UTL_decodeRange(char *range, set &ranges) { char *ap = range; uint32 av = 0, bv = 0; while (*ap != 0) { av = strtoull(ap, &ap, 10); if (*ap == ',') { ap++; ranges.insert(av); } else if (*ap == 0) { ranges.insert(av); } else if (*ap == '-') { ap++; bv = strtoull(ap, &ap, 10); for (uint32 xx=av; xx<=bv; xx++) ranges.insert(xx); if (*ap == ',') ap++; } else if (*ap != 0) { fprintf(stderr, "ERROR: invalid range '%s'\n", range); exit(1); } } } void AS_UTL_decodeRange(char *range, set &ranges) { char *ap = range; uint32 av = 0, bv = 0; while (*ap != 0) { av = strtoul(ap, &ap, 10); if (*ap == ',') { ap++; ranges.insert(av); } else if (*ap == 0) { ranges.insert(av); } else if (*ap == '-') { ap++; bv = strtoull(ap, &ap, 10); for (uint32 xx=av; xx<=bv; xx++) ranges.insert(xx); if (*ap == ',') ap++; } else if (*ap != 0) { fprintf(stderr, "ERROR: invalid range '%s'\n", range); exit(1); } } } void AS_UTL_decodeRange(char *range, uint64 &lo, uint64 &hi) { char *ap = range; lo = hi = strtoull(ap, &ap, 10); if (*ap == '-') { ap++; hi = strtoull(ap, &ap, 10); } else if (*ap != 0) { fprintf(stderr, "ERROR: invalid range '%s'\n", range); exit(1); } } void AS_UTL_decodeRange(char *range, int64 &lo, int64 &hi) { char *ap = range; lo = hi = strtoll(ap, &ap, 10); if (*ap == '-') { ap++; hi = strtoll(ap, &ap, 10); } else if (*ap != 0) { fprintf(stderr, "ERROR: invalid range '%s'\n", range); exit(1); } } void AS_UTL_decodeRange(char *range, uint32 &lo, uint32 &hi) { char *ap = range; lo = hi = strtoul(ap, &ap, 10); if (*ap == '-') { ap++; hi = strtoul(ap, &ap, 10); } else if (*ap != 0) { fprintf(stderr, "ERROR: invalid range '%s'\n", range); exit(1); } } void AS_UTL_decodeRange(char *range, int32 &lo, int32 &hi) { char *ap = range; lo = hi = strtol(ap, &ap, 10); if (*ap == '-') { ap++; hi = strtol(ap, &ap, 10); } else if (*ap != 0) { fprintf(stderr, "ERROR: invalid range '%s'\n", range); exit(1); } } void AS_UTL_decodeRange(char *range, double &lo, double &hi) { char *ap = range; lo = hi = strtod(ap, &ap); if (*ap == '-') { ap++; hi = strtod(ap, &ap); } else if (*ap != 0) { fprintf(stderr, "ERROR: invalid range '%s'\n", range); exit(1); } } canu-1.3/src/AS_UTL/AS_UTL_decodeRange.H000066400000000000000000000032261272335422300175470ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2012-FEB-12 to 2013-OCT-11 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2015-MAY-28 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef AS_UTL_DECODERANGE_H #define AS_UTL_DECODERANGE_H #include "AS_global.H" #include using namespace std; void AS_UTL_decodeRange(char *range, set &ranges); void AS_UTL_decodeRange(char *range, set &ranges); void AS_UTL_decodeRange(char *range, uint64 &lo, uint64 &hi); void AS_UTL_decodeRange(char *range, int64 &lo, int64 &hi); void AS_UTL_decodeRange(char *range, uint32 &lo, uint32 &hi); void AS_UTL_decodeRange(char *range, int32 &lo, int32 &hi); void AS_UTL_decodeRange(char *range, double &lo, double &hi); #endif // AS_UTL_DECODERANGE_H canu-1.3/src/AS_UTL/AS_UTL_fasta.C000066400000000000000000000054151272335422300164420ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2007-NOV-02 to 2013-AUG-01 * are Copyright 2007-2008,2010,2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-JUN-04 to 2009-MAR-31 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2013-MAR-20 * are Copyright 2013 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz on 2015-FEB-27 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-23 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_UTL_fasta.H" #include "AS_UTL_fileIO.H" #include void AS_UTL_writeFastA(FILE *f, char *s, int sl, int bl, char *h, ...) { va_list ap; char *o = new char [sl + sl / 60 + 2]; int si = 0; int oi = 0; while (si < sl) { o[oi++] = s[si++]; if (bl != 0 && (si % bl) == 0) o[oi++] = '\n'; } if (o[oi-1] != '\n') o[oi++] = '\n'; o[oi] = 0; va_start(ap, h); vfprintf(f, h, ap); va_end(ap); AS_UTL_safeWrite(f, o, "AS_UTL_writeFastA", sizeof(char), oi); delete [] o; } void AS_UTL_writeFastQ(FILE *f, char *s, int sl, char *q, int ql, char *h, ...) { va_list ap; char *o = new char [ql + 1]; int qi = 0; int oi = 0; assert(sl == ql); // Reencode the QV to the Sanger spec. This is a copy, so we don't need to undo it. while (qi < ql) o[oi++] = q[qi++] + '!'; o[oi] = 0; va_start(ap, h); vfprintf(f, h, ap); va_end(ap); AS_UTL_safeWrite(f, s, "AS_UTL_writeFastQ", sizeof(char), sl); fprintf(f, "\n"); fprintf(f, "+\n"); AS_UTL_safeWrite(f, o, "AS_UTL_writeFastQ", sizeof(char), ql); fprintf(f, "\n"); delete [] o; } canu-1.3/src/AS_UTL/AS_UTL_fasta.H000066400000000000000000000032641272335422300164470ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2007-NOV-02 to 2013-AUG-01 * are Copyright 2007-2008,2010,2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-OCT-29 to 2009-MAR-31 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz beginning on 2015-NOV-23 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef AS_UTL_FASTA_H #define AS_UTL_FASTA_H #include "AS_global.H" // Writes sequence as fasta, with at most 'bl' letters per line (unlimited if 0). void AS_UTL_writeFastA(FILE *f, char *s, int sl, int bl, char *h, ...); // Writes FastQ, with Sanger QVs. void AS_UTL_writeFastQ(FILE *f, char *s, int sl, char *q, int ql, char *h, ...); #endif canu-1.3/src/AS_UTL/AS_UTL_fileIO.C000066400000000000000000000321321272335422300165070ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2005-SEP-26 to 2013-SEP-27 * are Copyright 2005-2009,2011-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Eli Venter on 2007-AUG-27 * are Copyright 2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2015-MAR-25 to 2015-AUG-14 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_UTL_fileIO.H" // Report ALL attempts to seek somewhere. #undef DEBUG_SEEK // Use ftell() to verify that we wrote the expected number of bytes, // and that we ended up at the expected location. #undef VERIFY_WRITE_POSITIONS // Provides a safe and reliable mechanism for reading / writing // binary data. // // Split writes/reads into smaller pieces, check the result of each // piece. Really needed by OSF1 (V5.1), useful on other platforms to // be a little more friendly (big writes are usually not // interruptable). void AS_UTL_safeWrite(FILE *file, const void *buffer, const char *desc, size_t size, size_t nobj) { size_t position = 0; size_t length = 32 * 1024 * 1024 / size; size_t towrite = 0; size_t written = 0; #ifdef VERIFY_WRITE_POSITIONS off_t expectedposition = AS_UTL_ftell(file) + nobj * size; if (errno) // If we return, and errno is set, the stream isn't seekable. expectedposition = 0; #endif while (position < nobj) { towrite = length; if (position + towrite > nobj) towrite = nobj - position; errno = 0; written = fwrite(((char *)buffer) + position * size, size, towrite, file); if (errno) { fprintf(stderr, "safeWrite()-- Write failure on %s: %s\n", desc, strerror(errno)); fprintf(stderr, "safeWrite()-- Wanted to write "F_SIZE_T" objects (size="F_SIZE_T"), wrote "F_SIZE_T".\n", towrite, size, written); assert(errno == 0); } position += written; } // This catches a bizarre bug on FreeBSD (6.1 for sure, 4.10 too, I // think) where we write at the wrong location; see fseek below. // // UNFORTUNATELY, you can't ftell() on stdio. // #ifdef VERIFY_WRITE_POSITIONS if ((expectedposition > 0) && (AS_UTL_ftell(file) != expectedposition)) { fprintf(stderr, "safeWrite()-- EXPECTED "F_OFF_T", ended up at "F_OFF_T"\n", expectedposition, AS_UTL_ftell(file)); assert(AS_UTL_ftell(file) == expectedposition); } #endif } size_t AS_UTL_safeRead(FILE *file, void *buffer, const char *desc, size_t size, size_t nobj) { size_t position = 0; size_t length = 32 * 1024 * 1024 / size; size_t toread = 0; size_t written = 0; // readen? while (position < nobj) { toread = length; if (position + toread > nobj) toread = nobj - position; errno = 0; written = fread(((char *)buffer) + position * size, size, toread, file); position += written; if (feof(file) || (written == 0)) goto finish; if ((errno) && (errno != EINTR)) { fprintf(stderr, "safeRead()-- Read failure on %s: %s.\n", desc, strerror(errno)); fprintf(stderr, "safeRead()-- Wanted to read "F_SIZE_T" objects (size="F_SIZE_T"), read "F_SIZE_T".\n", toread, size, written); assert(errno == 0); } } finish: // Just annoys developers. Stop it. //if (position != nobj) // fprintf(stderr, "AS_UTL_safeRead()-- Short read; wanted "F_SIZE_T" objects, read "F_SIZE_T" instead.\n", // nobj, position); return(position); } // Ensure that directory 'dirname' exists. Returns true if the // directory needed to be created, false if it already exists. int AS_UTL_mkdir(const char *dirname) { struct stat st; errno = 0; // Stat the file. Don't fail if the file doesn't exist though. if ((stat(dirname, &st) != 0) && (errno != ENOENT)) { fprintf(stderr, "AS_UTL_mkdir()-- Couldn't stat '%s': %s\n", dirname, strerror(errno)); exit(1); } // If file doesn't exist, make the directory, and fail horribly, or return success. if (errno == ENOENT) { if (mkdir(dirname, S_IRWXU | S_IRWXG | S_IRWXO) != 0) { fprintf(stderr, "AS_UTL_mkdir()-- Couldn't create directory '%s': %s\n", dirname, strerror(errno)); exit(1); } return(1); } // The file exists. Return that it already exists, or that it's a file and fail horribly. if (S_ISDIR(st.st_mode)) return(0); fprintf(stderr, "AS_UTL_mkdir()-- ERROR! '%s' is a file, and not a directory.\n", dirname); exit(1); return(1); } // Remove a file, or do nothing if the file doesn't exist. Returns true if the file // was deleted, false if the file never existsed. int AS_UTL_unlink(const char *filename) { if (AS_UTL_fileExists(filename, FALSE, FALSE) == 0) return(0); errno = 0; unlink(filename); if (errno) { fprintf(stderr, "AS_UTL_unlink()-- Failed to remove file '%s': %s\n", filename, strerror(errno)); exit(1); } return(1); } // Returns true if the named file/directory exists, and permissions // allow us to read and/or write. // int AS_UTL_fileExists(const char *path, int directory, int readwrite) { struct stat s; int r; errno = 0; r = stat(path, &s); if (errno) return(0); if ((directory == 1) && (readwrite == 0) && (s.st_mode & S_IFDIR) && (s.st_mode & (S_IRUSR | S_IRGRP | S_IROTH)) && (s.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) return(1); if ((directory == 1) && (readwrite == 1) && (s.st_mode & S_IFDIR) && (s.st_mode & (S_IRUSR | S_IRGRP | S_IROTH)) && (s.st_mode & (S_IWUSR | S_IWGRP | S_IWOTH)) && (s.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) return(1); if ((directory == 0) && (readwrite == 0) && (s.st_mode & (S_IRUSR | S_IRGRP | S_IROTH))) return(1); if ((directory == 0) && (readwrite == 1) && (s.st_mode & (S_IRUSR | S_IRGRP | S_IROTH)) && (s.st_mode & (S_IWUSR | S_IWGRP | S_IWOTH))) return(1); return(0); } off_t AS_UTL_sizeOfFile(const char *path) { struct stat s; int r; off_t size = 0; errno = 0; r = stat(path, &s); if (errno) { fprintf(stderr, "Failed to stat() file '%s': %s\n", path, strerror(errno)); exit(1); } // gzipped files contain a file contents list, which we can // use to get the uncompressed size. // // gzip -l // compressed uncompressed ratio uncompressed_name // 14444 71680 79.9% up.tar // // bzipped files have no contents and we just guess. if (strcasecmp(path+strlen(path)-3, ".gz") == 0) { char cmd[256]; FILE *F; sprintf(cmd, "gzip -l %s", path); F = popen(cmd, "r"); fscanf(F, " %*s %*s %*s %*s "); fscanf(F, " %*d "F_OFF_T" %*s %*s ", &size); pclose(F); } else if (strcasecmp(path+strlen(path)-4, ".bz2") == 0) { size = s.st_size * 14 / 10; } else { size = s.st_size; } return(size); } off_t AS_UTL_ftell(FILE *stream) { off_t pos = 0; errno = 0; pos = ftello(stream); if ((errno == ESPIPE) || (errno == EBADF)) // Not a seekable stream. Return some goofy big number. return(((off_t)1) < 42); if (errno) fprintf(stderr, "AS_UTL_ftell()-- Failed with %s.\n", strerror(errno)); assert(errno == 0); return(pos); } void AS_UTL_fseek(FILE *stream, off_t offset, int whence) { off_t beginpos = AS_UTL_ftell(stream); // If the stream is already at the correct position, just return. // // Unless we're on FreeBSD. For unknown reasons, FreeBSD fails // updating the gkpStore with mate links. It seems to misplace the // file pointer, and ends up writing the record to the wrong // location. ftell() is returning the correct current location, // and so AS_PER_genericStore doesn't seek() and just writes to the // current position. At the end of the write, we're off by 4096 // bytes. // // LINK 498318175,1538 <-> 498318174,1537 // AS_UTL_fseek()-- seek to 159904 (whence=0); already there // safeWrite()-- write nobj=1x104 = 104 bytes at position 159904 // safeWrite()-- wrote nobj=1x104 = 104 bytes position now 164000 // safeWrite()-- EXPECTED 160008, ended up at 164000 // #if !defined __FreeBSD__ && !defined __osf__ && !defined __APPLE__ if ((whence == SEEK_SET) && (beginpos == offset)) { #ifdef DEBUG_SEEK // This isn't terribly informative, and adds a lot of clutter. //fprintf(stderr, "AS_UTL_fseek()-- seek to "F_OFF_T" (whence=%d); already there\n", offset, whence); #endif return; } #endif // __FreeBSD__ if (fseeko(stream, offset, whence) != 0) { fprintf(stderr, "AS_UTL_fseek()-- Failed with %s.\n", strerror(errno)); assert(errno == 0); } #ifdef DEBUG_SEEK fprintf(stderr, "AS_UTL_fseek()-- seek to "F_OFF_T" (requested "F_OFF_T", whence=%d) from "F_OFF_T"\n", AS_UTL_ftell(stream), offset, whence, beginpos); #endif if (whence == SEEK_SET) assert(AS_UTL_ftell(stream) == offset); } void AS_UTL_loadFileList(char *fileName, vector &fileList) { errno = 0; FILE *F = fopen(fileName, "r"); if (errno) fprintf(stderr, "Can't open '%s': %s\n", fileName, strerror(errno)), exit(1); char *line = new char [FILENAME_MAX]; fgets(line, FILENAME_MAX, F); while (!feof(F)) { chomp(line); fileList.push_back(line); line = new char [FILENAME_MAX]; fgets(line, FILENAME_MAX, F); } delete [] line; fclose(F); } compressedFileReader::compressedFileReader(const char *filename) { char cmd[FILENAME_MAX * 2]; int32 len = 0; _file = NULL; _pipe = false; _stdi = false; if (filename != NULL) len = strlen(filename); if ((len > 0) && (strcmp(filename, "-") != 0) && (AS_UTL_fileExists(filename, FALSE, FALSE) == FALSE)) fprintf(stderr, "ERROR: Failed to open input file '%s': %s\n", filename, strerror(errno)), exit(1); errno = 0; if ((len > 3) && (strcasecmp(filename + len - 3, ".gz") == 0)) { sprintf(cmd, "gzip -dc %s", filename); _file = popen(cmd, "r"); _pipe = true; } else if ((len > 4) && (strcasecmp(filename + len - 4, ".bz2") == 0)) { sprintf(cmd, "bzip2 -dc %s", filename); _file = popen(cmd, "r"); _pipe = true; } else if ((len > 3) && (strcasecmp(filename + len - 3, ".xz") == 0)) { sprintf(cmd, "xz -dc %s", filename); _file = popen(cmd, "r"); _pipe = true; if (_file == NULL) // popen() returns NULL on error. It does not reliably set errno. fprintf(stderr, "ERROR: Failed to open input file '%s': popen() returned NULL\n", filename), exit(1); errno = 0; } else if ((len == 0) || (strcmp(filename, "-") == 0)) { _file = stdin; _stdi = 1; } else { _file = fopen(filename, "r"); _pipe = false; } if (errno) fprintf(stderr, "ERROR: Failed to open input file '%s': %s\n", filename, strerror(errno)), exit(1); } compressedFileReader::~compressedFileReader() { if (_file == NULL) return; if (_stdi) return; if (_pipe) pclose(_file); else fclose(_file); } compressedFileWriter::compressedFileWriter(const char *filename, int32 level) { char cmd[FILENAME_MAX * 2]; int32 len = 0; _file = NULL; _pipe = false; _stdi = false; if (filename != NULL) len = strlen(filename); errno = 0; if ((len > 3) && (strcasecmp(filename + len - 3, ".gz") == 0)) { sprintf(cmd, "gzip -%dc > %s", level, filename); _file = popen(cmd, "w"); _pipe = true; } else if ((len > 4) && (strcasecmp(filename + len - 4, ".bz2") == 0)) { sprintf(cmd, "bzip2 -%dc > %s", level, filename); _file = popen(cmd, "w"); _pipe = true; } else if ((len > 3) && (strcasecmp(filename + len - 3, ".xz") == 0)) { sprintf(cmd, "xz -%dc > %s", level, filename); _file = popen(cmd, "w"); _pipe = true; } else if ((len == 0) || (strcmp(filename, "-") == 0)) { _file = stdout; _stdi = 1; } else { _file = fopen(filename, "w"); _pipe = false; } if (errno) fprintf(stderr, "ERROR: Failed to open output file '%s': %s\n", filename, strerror(errno)), exit(1); } compressedFileWriter::~compressedFileWriter() { if (_file == NULL) return; if (_stdi) return; if (_pipe) pclose(_file); else fclose(_file); } canu-1.3/src/AS_UTL/AS_UTL_fileIO.H000066400000000000000000000057771272335422300165330ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2005-SEP-26 to 2013-SEP-27 * are Copyright 2005,2007-2009,2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2015-AUG-14 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef AS_UTL_FILEIO_H #define AS_UTL_FILEIO_H #include "AS_global.H" #include using namespace std; // Provides a safe and reliable mechanism for reading / writing // binary data. // // Split writes/reads into smaller pieces, check the result of each // piece. Really needed by OSF1 (V5.1), useful on other platforms to // be a little more friendly (big writes are usually not // interruptable). #ifndef O_LARGEFILE #define O_LARGEFILE 0 #endif void AS_UTL_safeWrite(FILE *file, const void *buffer, const char *desc, size_t size, size_t nobj); size_t AS_UTL_safeRead (FILE *file, void *buffer, const char *desc, size_t size, size_t nobj); int AS_UTL_mkdir(const char *dirname); int AS_UTL_unlink(const char *filename); int AS_UTL_fileExists(const char *path, int directory=false, int readwrite=false); off_t AS_UTL_sizeOfFile(const char *path); off_t AS_UTL_ftell(FILE *stream); void AS_UTL_fseek(FILE *stream, off_t offset, int whence); // Read a file-of-files into a vector void AS_UTL_loadFileList(char *fileName, vector &fileList); class compressedFileReader { public: compressedFileReader(char const *filename); ~compressedFileReader(); FILE *operator*(void) { return(_file); }; FILE *file(void) { return(_file); }; bool isCompressed(void) { return(_pipe); }; private: FILE *_file; bool _pipe; bool _stdi; }; class compressedFileWriter { public: compressedFileWriter(char const *filename, int32 level=1); ~compressedFileWriter(); FILE *operator*(void) { return(_file); }; FILE *file(void) { return(_file); }; bool isCompressed(void) { return(_pipe); }; private: FILE *_file; bool _pipe; bool _stdi; }; #endif // AS_UTL_FILEIO_H canu-1.3/src/AS_UTL/AS_UTL_reverseComplement.C000066400000000000000000000063751272335422300210510ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2008-MAY-15 to 2013-AUG-01 * are Copyright 2008-2009,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" static char inv[256] = {0}; static void initRC(void) { if (inv['a'] == 't') return; inv['a'] = 't'; inv['c'] = 'g'; inv['g'] = 'c'; inv['t'] = 'a'; inv['n'] = 'n'; inv['A'] = 'T'; inv['C'] = 'G'; inv['G'] = 'C'; inv['T'] = 'A'; inv['N'] = 'N'; inv['-'] = '-'; } void reverseComplementSequence(char *seq, int len) { char c=0; char *s=seq, *S=seq+len-1; initRC(); if (len == 0) { len = strlen(seq); S = seq + len - 1; } while (s < S) { c = *s; *s++ = inv[*S]; *S-- = inv[c]; } if (s == S) *s = inv[*s]; } // Inplace reverse-complement an ACGT sequence. A pointer the the // string is returned. // #if 0 // From kmer char * reverseComplementSequence(char *seq, uint32 seqlen) { char *s = seq; char *e = seq + seqlen - 1; char t; uint32 c = seqlen / 2; while (c--) { t = complementSymbol[*s]; *(s++) = complementSymbol[*e]; *(e--) = t; } if (s == e) *s = complementSymbol[*s]; return(seq); } #endif void reverseComplement(char *seq, char *qlt, int len) { char c=0; char *s=seq, *S=seq+len-1; char *q=qlt, *Q=qlt+len-1; if (qlt == NULL) { reverseComplementSequence(seq, len); return; } initRC(); if (len == 0) { len = strlen(seq); S = seq + len - 1; Q = qlt + len - 1; } while (s < S) { c = *s; *s++ = inv[*S]; *S-- = inv[c]; c = *q; *q++ = *Q; *Q-- = c; } if (s == S) *s = inv[*s]; } void reverse(char *a, char *b, int len) { char c=0; char *s=a, *S=a+len-1; char *q=b, *Q=b+len-1; while (s < S) { c = *s; *s++ = *S; *S-- = c; c = *q; *q++ = *Q; *Q-- = c; } } // Inplace reverse a string. A pointer the the string is returned. // #if 0 // From kmer char * reverseString(char *seq, uint32 seqlen) { char *s = seq; char *e = seq + seqlen - 1; char t; uint32 c = seqlen / 2; while (c--) { t = *s; *(s++) = *e; *(e--) = t; } return(seq); } #endif canu-1.3/src/AS_UTL/AS_UTL_reverseComplement.H000066400000000000000000000024431272335422300210460ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2008-MAY-15 to 2013-AUG-01 * are Copyright 2008,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef REVERSECOMPLEMENT_H #define REVERSECOMPLEMENT_H #include "AS_global.H" void reverseComplementSequence(char *seq, int len); void reverseComplement(char *seq, char *qlt, int len); void reverse(char *a, char *b, int len); #endif canu-1.3/src/AS_UTL/AS_UTL_stackTrace.C000066400000000000000000000203171272335422300174260ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2013-MAR-14 to 2013-AUG-01 * are Copyright 2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-APR-26 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #ifndef __CYGWIN__ #ifndef _WIN32 #include // backtrace #endif #endif #include #include #include #include #include #include #include #include // Derived from // http://sourceware.org/git/?p=glibc.git;a=blob;f=debug/segfault.c // Linux g++ -rdynamic -g3 -o st1 st1.C // FreeBSD CC -o unwind unwind.C -I/usr/local/include -L/usr/local/lib -lunwind // (Not sure what this is about, but it was with the GDB commands at the end of this file) // // http://sourceware.org/gdb/onlinedocs/gdb/Separate-Debug-Files.html#Separate-Debug-Files // // objcopy --only-kep-debug foo foo.debug // strip -g foo // objcopy --add-gnu-debuglink=foo.debug foo #define WRITE_STRING(S) write(2, S, strlen(S)) void AS_UTL_envokeGDB(void) { #if 0 uint64 pid = getpid(); uint64 cid = fork(); char cmd[1024]; if (cid != 0) { // Parent waitpid(cid, NULL, 0); return; } // Child sprintf(cmd, "gdb -quiet -silent -p "F_U64" -batch -x commands", pid); system(cmd); exit(0); #endif } #ifdef LIBUNWIND #include void AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx) { WRITE_STRING("\nFailed with '"); WRITE_STRING(strsignal(sig_num)); WRITE_STRING("'\n"); WRITE_STRING("\nBacktrace (mangled):\n\n"); unw_cursor_t cursor; unw_context_t uc; unw_word_t ip, sp; int depth = 0; unw_getcontext(&uc); // Get state unw_init_local(&cursor, &uc); // Initialize state cursor while (unw_step(&cursor) > 0) { unw_get_reg(&cursor, UNW_REG_IP, &ip); unw_get_reg(&cursor, UNW_REG_SP, &sp); unw_word_t off; char name[256]; if (unw_get_proc_name (&cursor, name, sizeof (name), &off) == 0) { if (off) fprintf(stderr, "%02d <%s + 0x%lx> ip=%lx sp=%lx\n", depth, name, (long)off, ip, sp); else fprintf(stderr, "%02d <%s> ip=%lx sp=%lx\n", depth, name, ip, sp); } else { fprintf(stderr, "%02d ip=%lx sp=%lx\n", depth, ip, sp); } depth++; } //WRITE_STRING("\nBacktrace (demangled):\n\n"); //WRITE_STRING("\nGDB:\n\n"); //AS_UTL_envokeGDB(); // Pass the signal through, only so a core file can get generated. struct sigaction sa; sa.sa_handler = SIG_DFL; sigemptyset (&sa.sa_mask); sa.sa_flags = 0; sigaction(sig_num, &sa, NULL); raise(sig_num); } #elif defined(_GNU_SOURCE) void AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx) { void *arr[256]; #ifndef __CYGWIN__ #ifndef _WIN32 int32 cnt = backtrace(arr, 256); // Report the signal we failed on, be careful to not allocate memory. WRITE_STRING("\nFailed with '"); WRITE_STRING(strsignal(sig_num)); WRITE_STRING("'\n"); // Dump a full backtrace, even including the signal handler frames, instead of not dumping anything. // The _fd version doesn't allocate memory, the standard backtrace_symbols does. Demangling // names also allocates memory. We'll do the safe one first, just to get something, then try the // others. WRITE_STRING("\nBacktrace (mangled):\n\n"); backtrace_symbols_fd(arr, cnt, 2); // Now that we have something, try to generate a pretty backtrace. WRITE_STRING("\nBacktrace (demangled):\n\n"); char **messages = backtrace_symbols(arr, cnt); for (int32 i=0; (i < cnt) && (messages != NULL); ++i) { char *mang = NULL; char *obgn = NULL; char *oend = NULL; // find parantheses and +address offset surrounding mangled name for (char *p=messages[i]; (*p) && (oend == NULL); ++p) { if (*p == '(') mang = p; else if (*p == '+') obgn = p; else if (*p == ')') oend = p; } // If all three are valid, attempt to demangle the symbol if ((mang != NULL) && (obgn != NULL) && (oend != NULL) && (mang < obgn)) { *mang++ = 0; *obgn++ = 0; *oend++ = 0; int32 status = 0; char *name = abi::__cxa_demangle(mang, 0, 0, &status); if (status > 0) // Failed. name = mang; fprintf(stderr, "[%d] %s::%s + %s %s\n", i, messages[i], name, obgn, oend); //free(name); } else { fprintf(stderr, "[%d] %s\n", i, messages[i]); } } WRITE_STRING("\nGDB:\n\n"); AS_UTL_envokeGDB(); WRITE_STRING("\n"); #endif #endif // Pass the signal through, only so a core file can get generated. struct sigaction sa; sa.sa_handler = SIG_DFL; sigemptyset (&sa.sa_mask); sa.sa_flags = 0; sigaction(sig_num, &sa, NULL); raise(sig_num); } #endif void AS_UTL_installCrashCatcher(void) { #if (!defined(LIBUNWIND) && !defined(_GNU_SOURCE)) // No handler, just get out of here. return; #else struct sigaction sigact; memset(&sigact, 0, sizeof(struct sigaction)); sigact.sa_sigaction = AS_UTL_catchCrash; sigact.sa_flags = SA_RESTART | SA_SIGINFO; // Don't especially care if these fail or not. //sigaction(SIGINT, &sigact, NULL); // Interrupt sigaction(SIGILL, &sigact, NULL); // Illegal instruction sigaction(SIGFPE, &sigact, NULL); // Floating point exception sigaction(SIGABRT, &sigact, NULL); // Abort - from assert sigaction(SIGBUS, &sigact, NULL); // Bus error sigaction(SIGSEGV, &sigact, NULL); // Segmentation fault #endif } // // Proposed commands for GDB. // // print "" // print "info thread" // print "" // info thread // # // print "" // print "info proc all" // print "" // info proc all // # // print "" // print "maintenance info sections" // print "" // maintenance info sections // # // print "" // print "where" // print "" // where // # // print "" // print "" // print "" // info args // # // print "" // print "" // print "" // info locals // # // # // print "" // print "" // print "" // up // # // # // print "" // print "" // print "" // info args // # // print "" // print "" // print "" // info locals // up // # // # // print "" // print "" // print "" // info args // # // print "" // print "" // print "" // info locals // up // # // # // print "" // print "" // print "" // info args // # // print "" // print "" // print "" // info locals // up // # // # // print "" // print "" // print "" // info args // # // print "" // print "" // print "" // info locals // up // # // # // print "" // print "" // print "" // info args // # // print "" // print "" // print "" // info locals // up // # // # // print "" // print "" // print "" // info args // # // print "" // print "" // print "" // info locals // up // # // # // print "" // print "" // print "" // info args // # // print "" // print "" // print "" // info locals // up // # // # // print "" // print "" // print "" // info args // # // print "" // print "" // print "" // info locals // up // # // # // print "" // print "" // print "" // info args // # // print "" // print "" // print "" // info locals // up // # // quit // canu-1.3/src/AS_UTL/AS_UTL_stackTrace.H000066400000000000000000000024111272335422300174260ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2013-MAR-14 to 2013-AUG-01 * are Copyright 2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef AS_UTL_STACKTRACE_H #define AS_UTL_STACKTRACE_H #include void AS_UTL_catchCrash(int sig_num, siginfo_t *info, void *ctx); void AS_UTL_installCrashCatcher(void); #endif // AS_UTL_STACKTRACE_H canu-1.3/src/AS_UTL/AS_UTL_stackTraceTest.C000066400000000000000000000025671272335422300202750ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2013-OCT-18 * are Copyright 2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ int crash() { char * p = NULL; *p = 0; return 0; } int foo4() { crash(); return 0; } int foo3() { foo4(); return 0; } int foo2() { foo3(); return 0; } int foo1() { foo2(); return 0; } int main(int argc, char ** argv) { struct sigaction sigact; sigact.sa_sigaction = crit_err_hdlr; sigact.sa_flags = SA_RESTART | SA_SIGINFO; // Don't especially care if this fails or not. sigaction(SIGSEGV, &sigact, NULL); foo1(); exit(EXIT_SUCCESS); } canu-1.3/src/AS_UTL/bitEncodings.C000066400000000000000000000107521272335422300166450ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "bitEncodings.H" const uint32 fibonacciValuesLen = 92; const uint64 fibonacciValues[92] = { 1LLU, 2LLU, 3LLU, 5LLU, 8LLU, 13LLU, 21LLU, 34LLU, 55LLU, 89LLU, 144LLU, 233LLU, 377LLU, 610LLU, 987LLU, 1597LLU, 2584LLU, 4181LLU, 6765LLU, 10946LLU, 17711LLU, 28657LLU, 46368LLU, 75025LLU, 121393LLU, 196418LLU, 317811LLU, 514229LLU, 832040LLU, 1346269LLU, 2178309LLU, 3524578LLU, 5702887LLU, 9227465LLU, 14930352LLU, 24157817LLU, 39088169LLU, 63245986LLU, 102334155LLU, 165580141LLU, 267914296LLU, 433494437LLU, 701408733LLU, 1134903170LLU, 1836311903LLU, 2971215073LLU, 4807526976LLU, 7778742049LLU, 12586269025LLU, 20365011074LLU, 32951280099LLU, 53316291173LLU, 86267571272LLU, 139583862445LLU, 225851433717LLU, 365435296162LLU, 591286729879LLU, 956722026041LLU, 1548008755920LLU, 2504730781961LLU, 4052739537881LLU, 6557470319842LLU, 10610209857723LLU, 17167680177565LLU, 27777890035288LLU, 44945570212853LLU, 72723460248141LLU, 117669030460994LLU, 190392490709135LLU, 308061521170129LLU, 498454011879264LLU, 806515533049393LLU, 1304969544928657LLU, 2111485077978050LLU, 3416454622906707LLU, 5527939700884757LLU, 8944394323791464LLU, 14472334024676221LLU, 23416728348467685LLU, 37889062373143906LLU, 61305790721611591LLU, 99194853094755497LLU, 160500643816367088LLU, 259695496911122585LLU, 420196140727489673LLU, 679891637638612258LLU, 1100087778366101931LLU, 1779979416004714189LLU, 2880067194370816120LLU, 4660046610375530309LLU, 7540113804746346429LLU, 12200160415121876738LLU }; canu-1.3/src/AS_UTL/bitEncodings.H000066400000000000000000000240721272335422300166520ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "bitPacking.H" #include "bitOperations.H" // Methods to encode and decode numbers from an array of bits. // // The bit packed array is rooted at location 'ptr' and currently at location // 'pos'. The size of the encoded number (in bits) is returned in 'siz'. // // Came from kmer/libutil/ // eliasDeltaEncoding.h // eliasGammaEncoding.h // fibonacciEncoding.h // generalizedUnaryEncoding.h // unaryEncoding.h // unaryEncodingTester.C // // Unary Encoding // // Routines to store and retrieve a unary encoded number to/from a // bit packed word array based at 'ptr' and currently at location // 'pos'. Both routines return the size of the encoded number in // 'siz'. // // The usual unary encoding. Store the number n as n 0 bits followed // by a single 1 bit. // // 0 -> 1 // 1 -> 01 // 2 -> 001 // 3 -> 0001 // 4 -> 00001 // // See the decoder as to why we use 0 instead of 1 for the count. // inline void setUnaryEncodedNumber(uint64 *ptr, uint64 pos, uint64 *siz, uint64 val) { *siz = val + 1; while (val >= 64) { setDecodedValue(ptr, pos, 64, uint64ZERO); pos += 64; val -= 64; siz += 64; } setDecodedValue(ptr, pos, val + 1, uint64ONE); pos += val + 1; } inline uint64 getUnaryEncodedNumber(uint64 *ptr, uint64 pos, uint64 *siz) { uint64 val = uint64ZERO; uint64 enc = uint64ZERO; // How many whole words are zero? // enc = getDecodedValue(ptr, pos, 64); while (enc == uint64ZERO) { val += 64; pos += 64; enc = getDecodedValue(ptr, pos, 64); } // This word isn't zero. Count how many bits are zero (see, the // choice of 0 or 1 for the encoding wasn't arbitrary!) val += 64 - logBaseTwo64(enc); *siz = val + 1; return(val); } // // Generalized Unary Encoding // // Generalized unary encodings. Defined by (start, step, stop). // This implementation uses stop=infinity to encode all possible // numbers. If you know the highest number possible, you'll get a // slight decrease in space used ... // // The method: // // The mth code word consists of 'm' unary encoded, followed by w = // start + m * step binary encoded bits. If a == stop, then the // terminator in the unary code is dropped. // // Encoding is tricky. Take the 3,2,9 example: // m w template # vals #'s // 0 3 1xxx 8 0- 7 // 1 5 01xxxxx 32 8- 39 // 2 7 001xxxxxxx 128 40-167 // 3 9 000xxxxxxxxx 512 168-679 // // I don't see a nice way of mapping our number n to the prefix m, // short of some sort of search. The implementation below is // probably very slow. // // On the bright side, decoding is trivial. Read back the unary // encoded number, then read that many bits to get the value. // static const uint64 _genunary_start = 3; static const uint64 _genunary_step = 2; inline void setGeneralizedUnaryEncodedNumber(uint64 *ptr, uint64 pos, uint64 *siz, uint64 val) { uint64 m = uint64ZERO; uint64 w = _genunary_start; uint64 n = uint64ONE << w; // Search for the prefix m, given our number 'val'. // While doing this, we get rid of all the implicitly stored values from 'val'. // #ifdef DEBUG_GENERALIZEDUNARYENCODING fprintf(stderr, " val="uint64FMT" try n="uint64FMT" for m="uint64FMT"\n", val, n, m); #endif while (n <= val) { val -= n; w += _genunary_step; n = uint64ONE << w; m++; #ifdef DEBUG_GENERALIZEDUNARYENCODING fprintf(stderr, " val="uint64FMT" try n="uint64FMT" for m="uint64FMT"\n", val, n, m); #endif } #ifdef DEBUG_GENERALIZEDUNARYENCODING fprintf(stderr, "val="uint64FMT" found m="uint64FMT"\n", val, m); #endif // Now just encode the number // m - the unary encoded prefix // w - the size of the binary encoded number setUnaryEncodedNumber(ptr, pos, siz, m); setDecodedValue(ptr, pos+*siz, w, val); *siz = m + 1 + w; } inline uint64 getGeneralizedUnaryEncodedNumber(uint64 *ptr, uint64 pos, uint64 *siz) { uint64 val = uint64ZERO; uint64 m = uint64ZERO; uint64 w = uint64ZERO; // Comments in the encoder apply here too. m = getUnaryEncodedNumber(ptr, pos, siz); w = _genunary_start + m * _genunary_step; val = getDecodedValue(ptr, pos + *siz, w); *siz = m + 1 + w; #ifdef DEBUG_GENERALIZEDUNARYENCODING fprintf(stderr, "m="uint64FMT" w="uint64FMT" val="uint64FMT"\n", m, w, val); #endif // Add in the implcitly stored pieces of the number while (m--) { w -= _genunary_step; val += uint64ONE << w; } return(val); } // // Elias' Gamma Encoding // inline void setEliasGammaEncodedNumber(uint64 *ptr, uint64 pos, uint64 *siz, uint64 val) { uint64 b = logBaseTwo64(val); setUnaryEncodedNumber(ptr, pos, siz, b); pos += *siz; setDecodedValue(ptr, pos, b, val); *siz += b; } inline uint64 getEliasGammaEncodedNumber(uint64 *ptr, uint64 pos, uint64 *siz) { uint64 b = getUnaryEncodedNumber(ptr, pos, siz); pos += *siz; *siz += b; return(getDecodedValue(ptr, pos, b)); } // // Elias' Delta Encoding // inline void setEliasDeltaEncodedNumber(uint64 *ptr, uint64 pos, uint64 *siz, uint64 val) { uint64 b = logBaseTwo64(val); setEliasGammaEncodedNumber(ptr, pos, siz, b); pos += *siz; setDecodedValue(ptr, pos, b-1, val); *siz += b-1; } inline uint64 getEliasDeltaEncodedNumber(uint64 *ptr, uint64 pos, uint64 *siz) { uint64 b = getEliasGammaEncodedNumber(ptr, pos, siz) - 1; pos += *siz; *siz += b; return(uint64ONE << b | getDecodedValue(ptr, pos, b)); } // // Fibonacci Encoding // // FibEncoding can store values up to 17,167,680,177,565 (slightly // below 2^45, so at most a 44-bit number) in a 64-bit quantity. // 93 bits (92 + 1) are needed to store up to 64-bit values. // Since zero cannot be stored, the value is incremented by one before storing. // The actual space used is: // // #### bits // 0 2 // 1 3 // 2 4 // 3 4 // 4 5 // 5 5 // 6 5 // 7 6 // 8 6 // 9 6 // 10 6 // 11 6 // 12 7 // 20 8 // 33 9 // 54 10 // 88 11 // 143 12 // 232 13 // 376 14 // 609 15 // 986 16 // 1596 17 // 2583 18 // 4180 19 // 6764 20 // 10945 21 // 17710 22 // 28656 23 // 46387 24 // 75024 25 // 121392 26 extern const uint32 fibonacciValuesLen; extern const uint64 fibonacciValues[92]; inline void setFibonacciEncodedNumber(uint64 *ptr, uint64 pos, uint64 *siz, uint64 val) { uint64 out1 = uint64ZERO; uint64 out2 = uint64ZERO; uint32 fib = fibonacciValuesLen; uint32 fibmax = uint64ZERO; // We cannot store zero as a fibonacci number, so we simply // increase everything by one. // val++; // Estimate a starting point for our search; we need a function // that is always slightly more than fib() // // Find the highest bit set, do a lookup // // XXX: Still need this! while (fib-- > 0) { if (val >= fibonacciValues[fib]) { if (fib >= 64) out2 |= uint64ONE << (127 - fib); else out1 |= uint64ONE << (63 - fib); val -= fibonacciValues[fib]; if (fibmax == uint64ZERO) { fibmax = fib + 1; if (fibmax >= 64) out2 |= uint64ONE << (127 - fibmax); else out1 |= uint64ONE << (63 - fibmax); } } } fibmax++; // Write the encoded numbers to the stream // if (fibmax > 64) { setDecodedValue(ptr, pos, 64, out1); pos += 64; out2 >>= (128 - fibmax); setDecodedValue(ptr, pos, fibmax - 64, out2); } else { out1 >>= (64 - fibmax); setDecodedValue(ptr, pos, fibmax, out1); } *siz = fibmax; } inline uint64 getFibonacciEncodedNumber(uint64 *ptr, uint64 pos, uint64 *siz) { uint64 wrd = (pos >> 6) & 0x0000cfffffffffffllu; uint64 sft = 0x8000000000000000llu >> (pos & 0x000000000000003fllu); uint64 val = 0; uint32 fib = 0; uint64 newbit; uint64 oldbit; oldbit = ptr[wrd] & sft; sft >>= 1; if (sft == uint64ZERO) { wrd++; sft = 0x8000000000000000llu; } newbit = ptr[wrd] & sft; sft >>= 1; if (sft == uint64ZERO) { wrd++; sft = 0x8000000000000000llu; } while (!oldbit || !newbit) { if (oldbit) val += fibonacciValues[fib]; fib++; oldbit = newbit; newbit = ptr[wrd] & sft; sft >>= 1; if (sft == uint64ZERO) { wrd++; sft = 0x8000000000000000llu; } } val += fibonacciValues[fib]; (*siz) = fib + 2; // We stored val+1, remember? Probably not, because the encoder is // next. // return(val - 1); } canu-1.3/src/AS_UTL/bitOperations.H000066400000000000000000000145671272335422300170740ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/bitOperations.h * * Modifications by: * * Brian P. Walenz on 2004-APR-27 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-FEB-23 to 2014-APR-11 * are Copyright 2005-2006,2008,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef BRI_BITS_H #define BRI_BITS_H // For dealing with the bits in bytes. // I wish I could claim these. // // Freed, Edwin E. 1983. "Binary Magic Number" Dr. Dobbs Journal // Vol. 78 (April) pp. 24-37 // // Supposedly tells us how to reverse the bits in a word, count the number // of set bits in a words and more. // // A bit of verbage on counting the number of set bits. The naive way // is to loop and shift: // // uint32 r = uint32ZERO; // while (x) { // r++; // x >>= 1; // } // return(r); // // http://remus.rutgers.edu/~rhoads/Code/bitcount3.c has an optimized // method: // // x -= (0xaaaaaaaa & x) >> 1; // x = (x & 0x33333333) + ((x >> 2) & 0x33333333); // x += x >> 4; // x &= 0x0f0f0f0f; // x += x >> 8; // x += x >> 16; // x &= 0x000000ff; // return(x); // // No loops! // // Freed's methods are easier to understand, and just as fast. // // Using our bit counting routines, Ross Lippert suggested a nice // way of computing log2 -- use log2 shifts to fill up the lower // bits, then count bits. See logBaseTwo*() // inline uint64 reverseBits64(uint64 x) { x = ((x >> 1) & uint64NUMBER(0x5555555555555555)) | ((x << 1) & uint64NUMBER(0xaaaaaaaaaaaaaaaa)); x = ((x >> 2) & uint64NUMBER(0x3333333333333333)) | ((x << 2) & uint64NUMBER(0xcccccccccccccccc)); x = ((x >> 4) & uint64NUMBER(0x0f0f0f0f0f0f0f0f)) | ((x << 4) & uint64NUMBER(0xf0f0f0f0f0f0f0f0)); x = ((x >> 8) & uint64NUMBER(0x00ff00ff00ff00ff)) | ((x << 8) & uint64NUMBER(0xff00ff00ff00ff00)); x = ((x >> 16) & uint64NUMBER(0x0000ffff0000ffff)) | ((x << 16) & uint64NUMBER(0xffff0000ffff0000)); x = ((x >> 32) & uint64NUMBER(0x00000000ffffffff)) | ((x << 32) & uint64NUMBER(0xffffffff00000000)); return(x); } inline uint32 reverseBits32(uint32 x) { x = ((x >> 1) & uint32NUMBER(0x55555555)) | ((x << 1) & uint32NUMBER(0xaaaaaaaa)); x = ((x >> 2) & uint32NUMBER(0x33333333)) | ((x << 2) & uint32NUMBER(0xcccccccc)); x = ((x >> 4) & uint32NUMBER(0x0f0f0f0f)) | ((x << 4) & uint32NUMBER(0xf0f0f0f0)); x = ((x >> 8) & uint32NUMBER(0x00ff00ff)) | ((x << 8) & uint32NUMBER(0xff00ff00)); x = ((x >> 16) & uint32NUMBER(0x0000ffff)) | ((x << 16) & uint32NUMBER(0xffff0000)); return(x); } inline uint64 uint64Swap(uint64 x) { x = ((x >> 8) & uint64NUMBER(0x00ff00ff00ff00ff)) | ((x << 8) & uint64NUMBER(0xff00ff00ff00ff00)); x = ((x >> 16) & uint64NUMBER(0x0000ffff0000ffff)) | ((x << 16) & uint64NUMBER(0xffff0000ffff0000)); x = ((x >> 32) & uint64NUMBER(0x00000000ffffffff)) | ((x << 32) & uint64NUMBER(0xffffffff00000000)); return(x); } inline uint32 uint32Swap(uint32 x) { x = ((x >> 8) & uint32NUMBER(0x00ff00ff)) | ((x << 8) & uint32NUMBER(0xff00ff00)); x = ((x >> 16) & uint32NUMBER(0x0000ffff)) | ((x << 16) & uint32NUMBER(0xffff0000)); return(x); } inline uint16 uint16Swap(uint16 x) { x = ((x >> 8) & 0x00ff) | ((x << 8) & 0xff00); return(x); } #if (__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4) #define PREFETCH(x) __builtin_prefetch((x), 0, 0) #else #define PREFETCH(x) #endif // Amazingingly, this is slower. From what I can google, the builtin // is using the 2^16 lookup table method - so a 64-bit popcount does // 4 lookups in the table and sums. Bad cache performance in codes // that already have bad cache performance, I'd guess. // //#if (__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4) //#define BUILTIN_POPCOUNT //#endif #ifdef BUILTIN_POPCOUNT inline uint32 countNumberOfSetBits32(uint32 x) { return(__builtin_popcount(x)); } inline uint64 countNumberOfSetBits64(uint64 x) { return(__builtin_popcountll(x)); } #else inline uint32 countNumberOfSetBits32(uint32 x) { x = ((x >> 1) & uint32NUMBER(0x55555555)) + (x & uint32NUMBER(0x55555555)); x = ((x >> 2) & uint32NUMBER(0x33333333)) + (x & uint32NUMBER(0x33333333)); x = ((x >> 4) & uint32NUMBER(0x0f0f0f0f)) + (x & uint32NUMBER(0x0f0f0f0f)); x = ((x >> 8) & uint32NUMBER(0x00ff00ff)) + (x & uint32NUMBER(0x00ff00ff)); x = ((x >> 16) & uint32NUMBER(0x0000ffff)) + (x & uint32NUMBER(0x0000ffff)); return(x); } inline uint64 countNumberOfSetBits64(uint64 x) { x = ((x >> 1) & uint64NUMBER(0x5555555555555555)) + (x & uint64NUMBER(0x5555555555555555)); x = ((x >> 2) & uint64NUMBER(0x3333333333333333)) + (x & uint64NUMBER(0x3333333333333333)); x = ((x >> 4) & uint64NUMBER(0x0f0f0f0f0f0f0f0f)) + (x & uint64NUMBER(0x0f0f0f0f0f0f0f0f)); x = ((x >> 8) & uint64NUMBER(0x00ff00ff00ff00ff)) + (x & uint64NUMBER(0x00ff00ff00ff00ff)); x = ((x >> 16) & uint64NUMBER(0x0000ffff0000ffff)) + (x & uint64NUMBER(0x0000ffff0000ffff)); x = ((x >> 32) & uint64NUMBER(0x00000000ffffffff)) + (x & uint64NUMBER(0x00000000ffffffff)); return(x); } #endif inline uint32 logBaseTwo32(uint32 x) { x |= x >> 1; x |= x >> 2; x |= x >> 4; x |= x >> 8; x |= x >> 16; return(countNumberOfSetBits32(x)); } inline uint64 logBaseTwo64(uint64 x) { x |= x >> 1; x |= x >> 2; x |= x >> 4; x |= x >> 8; x |= x >> 16; x |= x >> 32; return(countNumberOfSetBits64(x)); } #endif // BRI_BITS_H canu-1.3/src/AS_UTL/bitPackedArray.C000066400000000000000000000067711272335422300171300ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/bitPackedArray.C * * Modifications by: * * Brian P. Walenz from 2005-FEB-07 to 2014-APR-11 * are Copyright 2005-2006,2012,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include #include #include #include "bitPackedArray.H" #include "bitPacking.H" bitPackedArray::bitPackedArray(uint32 valueWidth, uint32 segmentSize) { _valueWidth = valueWidth; _segmentSize = segmentSize; _nextElement = 0; _valuesPerSegment = (uint64)_segmentSize * 1024 * 8 / (uint64)_valueWidth; _numSegments = 0; _maxSegments = 16; _segments = new uint64 * [_maxSegments]; } bitPackedArray::~bitPackedArray() { for (uint32 i=0; i<_numSegments; i++) delete [] _segments[i]; delete [] _segments; } uint64 bitPackedArray::get(uint64 idx) { uint64 s = idx / _valuesPerSegment; uint64 p = _valueWidth * (idx % _valuesPerSegment); if (idx >= _nextElement) { fprintf(stderr, "bitPackedArray::get()-- element index "F_U64" is out of range, only "F_U64" elements.\n", idx, _nextElement-1); return(0xdeadbeefdeadbeefULL); } return(getDecodedValue(_segments[s], p, _valueWidth)); } void bitPackedArray::set(uint64 idx, uint64 val) { uint64 s = idx / _valuesPerSegment; uint64 p = _valueWidth * (idx % _valuesPerSegment); //fprintf(stderr, "s="F_U64" p="F_U64" segments="F_U64"/"F_U64"\n", s, p, _numSegments, _maxSegments); if (idx >= _nextElement) _nextElement = idx+1; if (s >= _maxSegments) { _maxSegments = s + 16; uint64 **S = new uint64 * [_maxSegments]; for (uint32 i=0; i<_numSegments; i++) S[i] = _segments[i]; delete [] _segments; _segments = S; } while (_numSegments <= s) _segments[_numSegments++] = new uint64 [_segmentSize * 1024 / 8]; setDecodedValue(_segments[s], p, _valueWidth, val); } void bitPackedArray::clear(void) { for (uint32 s=0; s<_numSegments; s++) bzero(_segments[s], _segmentSize * 1024); } //////////////////////////////////////// bitArray::bitArray(uint32 segmentSize) { _segmentSize = segmentSize; _valuesPerSegment = (uint64)_segmentSize * 1024 * 8; _numSegments = 0; _maxSegments = 16; _segments = new uint64 * [_maxSegments]; } bitArray::~bitArray() { for (uint32 i=0; i<_numSegments; i++) delete [] _segments[i]; delete [] _segments; } void bitArray::clear(void) { for (uint32 s=0; s<_numSegments; s++) bzero(_segments[s], _segmentSize * 1024); } canu-1.3/src/AS_UTL/bitPackedArray.H000066400000000000000000000177431272335422300171360ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/bitPackedArray.H * * Modifications by: * * Brian P. Walenz from 2005-JUL-12 to 2014-APR-11 * are Copyright 2005-2006,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef BITPACKEDARRAY_H #define BITPACKEDARRAY_H #include "AS_global.H" #undef DEBUG_BPH_ADD #undef DEBUG_BPH_GET //////////////////////////////////////// // // bitPackedArray // // implements an integer array using bit-widths less than word-sizes, // e.g., a memory efficient way to store 23 bit numbers. Numbers may // be up to 64 bits wide. // // The array is variable length, and it is implemented as an array, // not a list or tree -- accessing element 1,000,000 will allocate // elements 0 through 999,999. // class bitPackedArray { public: // Create a bitpacked array with elements of width 'width' using // 'segmentSize' KB per segment. If you know your array is going // to be much bigger or smaller, crank this value. // bitPackedArray(uint32 valueWidth, uint32 segmentSize = 1024); ~bitPackedArray(); // No array operator is provided, because we cannot return a // reference to a value that is split across two words (or even a // reference to a value that is not bit aligned in the word). // uint64 get(uint64 idx); void set(uint64 idx, uint64 val); // Clear the array. Since the array is variable sized, you must add // things to a new array before clearing it. void clear(void); private: uint32 _valueWidth; uint32 _segmentSize; uint64 _nextElement; // the first invalid element uint64 _valuesPerSegment; uint64 _numSegments; uint64 _maxSegments; uint64 **_segments; }; // An array of bits. Exactly the same as the bitPackedArray, but // optimized for width=1. // class bitArray { public: bitArray(uint32 segmentSize = 1024); ~bitArray(); uint64 get(uint64 idx); uint64 getAndSet(uint64 idx); void set(uint64 idx); void clr(uint64 idx); void clear(void); private: void resize(uint64 s); uint32 _segmentSize; uint64 _valuesPerSegment; uint64 _numSegments; uint64 _maxSegments; uint64 **_segments; }; // Uses the bitPackedArray to implement a heap. The bitPackedArray is dynamically sized, // so this can be too. // class bitPackedHeap { public: bitPackedHeap(uint32 width, uint64 size=16) { _array = new bitPackedArray(width, size); _array->set(0, 0); _lastVal = 0; }; ~bitPackedHeap() { delete _array; }; uint64 get(void) { uint64 biggestVal = ~uint64ZERO; if (_lastVal == 0) return(biggestVal); biggestVal = _array->get(0); _lastVal--; if (_lastVal == 0) return(biggestVal); uint64 t = _array->get(_lastVal); _array->set(0, t); uint64 pidx = 0; uint64 pval = t; uint64 cidx = 1; uint64 cval = 0; // set below while (cidx < _lastVal) { // Set cval here, so we can first test if cidx is in range. cval = _array->get(cidx); // Pick the smallest of the two kids if (cidx+1 < _lastVal) { t = _array->get(cidx+1); if (cval > t) { cidx++; cval = t; } } #ifdef DEBUG_BPH_GET fprintf(stderr, "test c="F_U64" and p="F_U64" lastVal="F_U64"\n", cidx, pidx, _lastVal); fprintf(stderr, "test c="F_U64"="F_U64"\n", cidx, cval); fprintf(stderr, "test p="F_U64"="F_U64"\n", pidx, pval); fprintf(stderr, "test c="F_U64"="F_U64" and p="F_U64"="F_U64"\n", cidx, cval, pidx, pval); #endif if (cval < pval) { #ifdef DEBUG_BPH_GET fprintf(stderr, "swap c="F_U64"="F_U64" and p="F_U64"="F_U64"\n", cidx, cval, pidx, pval); #endif // Swap p and c _array->set(pidx, cval); _array->set(cidx, pval); // Move down the tree -- pval doesn't change, we moved it into cidx! pidx = cidx; cidx = cidx * 2 + 1; } else { cidx = _lastVal; } } return(biggestVal); }; void add(uint64 value) { uint64 cidx = _lastVal; uint64 cval = value; uint64 pidx = 0; uint64 pval = 0; bool more = false; #ifdef DEBUG_BPH_ADD fprintf(stderr, "add c="F_U64"="F_U64" -- lastVal="F_U64"\n", cidx, cval, _lastVal); #endif _array->set(cidx, cval); if (cidx > 0) more = true; while (more) { pidx = (cidx-1) / 2; #ifdef DEBUG_BPH_ADD fprintf(stderr, "more c="F_U64" and p="F_U64"\n", cidx, pidx); #endif pval = _array->get(pidx); #ifdef DEBUG_BPH_ADD fprintf(stderr, "test c="F_U64"="F_U64" and p="F_U64"="F_U64"\n", cidx, cval, pidx, pval); #endif if (pval > cval) { #ifdef DEBUG_BPH_ADD fprintf(stderr, "swap c="F_U64"="F_U64" and p="F_U64"="F_U64"\n", cidx, cval, pidx, pval); #endif // Swap p and c _array->set(cidx, pval); _array->set(pidx, cval); // Move up the tree -- cval doesn't change, we moved it into pidx! cidx = pidx; } else { more = false; } if (cidx == 0) more = false; } _lastVal++; //dump(); }; void dump(void) { for (uint32 i=0; i<_lastVal; i++) fprintf(stderr, "HEAP["F_U32"]="F_U64"\n", i, _array->get(i)); } void clear(void) { _array->clear(); _lastVal = 0; }; private: bitPackedArray *_array; uint64 _lastVal; }; inline uint64 bitArray::get(uint64 idx) { uint64 s = idx / _valuesPerSegment; uint64 p = idx % _valuesPerSegment; uint64 wrd = (p >> 6) & 0x0000cfffffffffffllu; uint64 bit = (p ) & 0x000000000000003fllu; return((_segments[s][wrd] >> bit) & 0x0000000000000001llu); } inline void bitArray::resize(uint64 s) { if (s < _numSegments) return; if (s > _maxSegments) { _maxSegments = s + 16; uint64 **S = new uint64 * [_maxSegments]; for (uint32 i=0; i<_numSegments; i++) S[i] = _segments[i]; delete [] _segments; _segments = S; } while (_numSegments <= s) _segments[_numSegments++] = new uint64 [_segmentSize * 1024 / 8]; } inline uint64 bitArray::getAndSet(uint64 idx) { uint64 s = idx / _valuesPerSegment; uint64 p = idx % _valuesPerSegment; uint64 wrd = (p >> 6) & 0x0000cfffffffffffllu; uint64 bit = (p ) & 0x000000000000003fllu; uint64 ret = (_segments[s][wrd] >> bit) & 0x0000000000000001llu; _segments[s][wrd] |= uint64ONE << bit; return(ret); } inline void bitArray::set(uint64 idx) { uint64 s = idx / _valuesPerSegment; uint64 p = idx % _valuesPerSegment; resize(s); uint64 wrd = (p >> 6) & 0x0000cfffffffffffllu; uint64 bit = (p ) & 0x000000000000003fllu; _segments[s][wrd] |= uint64ONE << bit; } inline void bitArray::clr(uint64 idx) { uint64 s = idx / _valuesPerSegment; uint64 p = idx % _valuesPerSegment; resize(s); uint64 wrd = (p >> 6) & 0x0000cfffffffffffllu; uint64 bit = (p ) & 0x000000000000003fllu; _segments[s][wrd] &= ~(0x0000000000000001llu << bit); } #endif // BITPACKEDARRAY_H canu-1.3/src/AS_UTL/bitPackedFile.C000066400000000000000000000342571272335422300167310ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/bitPackedFile.C * * Modifications by: * * Brian P. Walenz from 2003-JAN-02 to 2004-APR-01 * are Copyright 2003-2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-MAR-29 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAR-16 to 2014-APR-11 * are Copyright 2005-2008,2012,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "bitPackedFile.H" #include "AS_UTL_fileIO.H" //#include //#include //#include //#include //#include #include // N.B. any read() / write() pair (either order) must have a seek (or // a fflush) in between. bitPackedFile::bitPackedFile(char const *name, uint64 offset, bool forceTruncate) { _file = 0; _name = new char [strlen(name) + 1]; strcpy(_name, name); #ifdef WITH_BZIP2 _bzFILE = 0L; _bzerr = 0; _bzfile = 0L; #endif _bfrmax = 1048576 / 8; _bfr = new uint64 [_bfrmax]; _pos = uint64ZERO; _bit = uint64ZERO; memset(_bfr, 0, sizeof(uint64) * _bfrmax); _inCore = false; _bfrDirty = false; _forceFirstLoad = false; _isReadOnly = false; _isBzip2 = false; stat_seekInside = uint64ZERO; stat_seekOutside = uint64ZERO; stat_dirtyFlushes = uint64ZERO; file_offset = 0; endianess_offset = 0; endianess_flipped = false; // Try to open the original name -- we don't support compressed // files for rewrite. We just fail with a can't open message. // // To get read/write and create we have to use open(2), as mode // "r+" of fopen(3) will not create. (Yes, but w+ does, sigh.) // if (forceTruncate) { errno = 0; _file = open(_name, O_RDWR | O_CREAT | O_TRUNC | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); if (errno) fprintf(stderr, "bitPackedFile::bitPackedFile()-- failed to open and truncate '%s': %s\n", _name, strerror(errno)), exit(1); } else if (AS_UTL_fileExists(_name)) { errno = 0; _file = open(_name, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); if (errno) fprintf(stderr, "bitPackedFile::bitPackedFile()-- failed to open '%s': %s\n", _name, strerror(errno)), exit(1); _isReadOnly = true; } else { errno = 0; _file = open(_name, O_RDWR | O_CREAT | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); if (errno) fprintf(stderr, "bitPackedFile::bitPackedFile()-- failed to open '%s': %s\n", _name, strerror(errno)), exit(1); } // Move to the correct position in the file. // file_offset = offset; if (file_offset > 0) lseek(_file, file_offset, SEEK_SET); // Deal with endianess. We write out some bytes (or read back some bytes) to the start of // the file, and then hide them from the user. // endianess_offset = 32 + file_offset; endianess_flipped = false; char t[16] = { 'b', 'i', 't', 'P', 'a', 'c', 'k', 'e', 'd', 'F', 'i', 'l', 'e', 0, 0, 1 }; char c[16] = { 0 }; uint64 at = uint64NUMBER(0xdeadbeeffeeddada ); uint64 bt = uint64NUMBER(0x0abeadedbabed8f8); uint64 ac = uint64NUMBER(0); uint64 bc = uint64NUMBER(0); size_t nr = 0; errno = 0; nr += read(_file, c, sizeof(char) * 16); nr += read(_file, &ac, sizeof(uint64)); nr += read(_file, &bc, sizeof(uint64)); if (nr == 0) { // Empty file! Write the magic number and our endianess check. errno = 0; write(_file, t, sizeof(char) * 16); write(_file, &at, sizeof(uint64)); write(_file, &bt, sizeof(uint64)); if (errno) fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' failed to write the header: %s\n", _name, strerror(errno)), exit(1); return; } if ((c[0] == 'B') && (c[1] == 'Z') && (c[2] == 'h')) { #ifdef WITH_BZIP2 // Looks like a bzip2 file! errno = 0; _bzFILE = fopen(_name, "r"); if (errno) { fprintf(stderr, "bitPackedFile::bitPackedFile()-- failed to open bzip2 file '%s'\n", _name); exit(1); } _bzerr = 0; _bzfile = BZ2_bzReadOpen(&_bzerr, _bzFILE, 0, 0, 0L, 0); if ((_bzfile == 0L) || (_bzerr != BZ_OK)) { fprintf(stderr, "bitPackedFile::bitPackedFile()-- failed to init bzip2 file '%s'\n", _name); exit(1); } BZ2_bzRead(&_bzerr, _bzfile, c, sizeof(char) * 16); BZ2_bzRead(&_bzerr, _bzfile, &ac, sizeof(uint64)); BZ2_bzRead(&_bzerr, _bzfile, &bc, sizeof(uint64)); // XXX should check bzerr! _isReadOnly = true; _isBzip2 = true; #else fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' looks like a bzip2 file, but bzip2 support not available!\n", _name); exit(1); #endif } // Check the magic number, decide on an endianess to use. // if (strncmp(t, c, 16) == 0) { if ((at == ac) && (bt == bc)) { endianess_flipped = false; } else if ((at == uint64Swap(ac)) && (bt == uint64Swap(bc))) { endianess_flipped = true; } else { fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' looked like a bitPackedFile, but failed the endianess check, not opened.\n", _name); exit(1); } } else { fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' doesn't appear to be a bitPackedFile, not opened.\n", _name); fprintf(stderr, "bitPackedFile::bitPackedFile()-- found "); for (uint32 i=0; i<16; i++) fprintf(stderr, "%c", isascii(c[i]) ? c[i] : '.'); fprintf(stderr, " at position "F_X64"\n", file_offset); exit(1); } _forceFirstLoad = true; seek(0); } bitPackedFile::~bitPackedFile() { flushDirty(); delete [] _bfr; delete [] _name; close(_file); #ifdef WITH_BZIP2 if (_bzFILE) fclose(_bzFILE); if (_bzfile) BZ2_bzReadClose(&_bzerr, _bzfile); #endif } // If the page is dirty, flush it to disk // void bitPackedFile::flushDirty(void) { if (_bfrDirty == false) return; if (_isReadOnly) { fprintf(stderr, "bitPackedFile::bitPackedFile()-- '%s' is readonly, but is dirty!\n", _name); exit(1); } stat_dirtyFlushes++; errno = 0; lseek(_file, _pos * sizeof(uint64) + endianess_offset, SEEK_SET); if (errno) { fprintf(stderr, "bitPackedFile::seek()-- '%s' failed: %s\n", _name, strerror(errno)); exit(1); } // If we need to, flip all the words we are going to write // if (endianess_flipped) for (uint32 i=0; i<_bfrmax; i++) _bfr[i] = uint64Swap(_bfr[i]); // We should only write bits up to _bit, the position we are // currently at. However, we don't know if the block is being // flushed because we're totally finished with it, or because we // are moving on to the next block. If we're done with it, we // want to flush the word that contains _bit, and if we're moving // on to the next one, we'll flush that word again. So, in // either case, we flush the word that contains _bit. // errno = 0; write(_file, _bfr, sizeof(uint64) * _bfrmax); if (errno) { fprintf(stderr, "bitPackedFile::write()-- '%s' failed: %s\n", _name, strerror(errno)); exit(1); } // And then flip them back // if (endianess_flipped) for (uint32 i=0; i<_bfrmax; i++) _bfr[i] = uint64Swap(_bfr[i]); _bfrDirty = false; } void bitPackedFile::seekBzip2(uint64 UNUSED(bitpos)) { #ifdef WITH_BZIP2 // All we can do here is check that bitpos is // a) in our current buffer // b) would be in the next buffer once we read it uint64 newpos = bitpos >> 6; if (_pos + _bfrmax < newpos) { // nope, not in the buffer -- we could probably handle this by just reading and // discarding from the file until we get to the correct bitpos. fprintf(stderr, "bitPackedFile::seekBzip2()-- '%s' seek was not contiguous!\n", _name); exit(1); } // Copy the remaining bits of the current buffer to the start. Or // not, if this is the first load. uint64 lastpos = _bit >> 6; // The word we are currently in uint64 lastlen = (_bfrmax - lastpos); // The number of words left in the buffer if (_forceFirstLoad == true) { lastpos = 0; lastlen = 0; } else { memcpy(_bfr, _bfr + lastpos, sizeof(uint64) * lastlen); } // Update _bit and _pos -- lastlen is now the first invalid word // _bit = bitpos & 0x3f; // 64 * lastlen; _pos = bitpos >> 6; // Fill the buffer size_t wordsread = 0; if (_bzfile) { _bzerr = 0; wordsread = BZ2_bzRead(&_bzerr, _bzfile, _bfr + lastlen, sizeof(uint64) * (_bfrmax - lastlen)); if (_bzerr == BZ_STREAM_END) { //fprintf(stderr, "bitPackedFile::seekBzip2() file ended.\n"); BZ2_bzReadClose(&_bzerr, _bzfile); fclose(_bzFILE); _bzfile = 0L; _bzFILE = 0L; } else if (_bzerr != BZ_OK) { fprintf(stderr, "bitPackedFile::seekBzip2() '%s' read failed.\n", _name); exit(1); } } //fprintf(stderr, "Filled buffer with %d words!\n", wordsread); // Adjust to make wordsread be the index of the last word we actually read. // wordsread += lastlen; // Flip all the words we just read, if needed // if (endianess_flipped) for (uint32 i=lastlen; i> 6) is just before the old // position (_pos), assume that we are being accessed iteratively // backwards and load a full buffer so that the position we want to // access is at the end. // // Easy to think of bone-headed ways to break this (e.g., seek to // the second element in a structure, access the first, then access // the third). Not so easy to think of a logical reason someone // would want to do that. // if (((bitpos >> 6) < _pos) && (_pos <= (bitpos >> 6) + 32)) { _pos = bitpos >> 6; if (_pos > _bfrmax) _pos = _pos - _bfrmax + 32; else _pos = 0; } else { _pos = bitpos >> 6; } _bit = bitpos - (_pos << 6); errno = 0; lseek(_file, _pos * 8 + endianess_offset, SEEK_SET); if (errno) { fprintf(stderr, "bitPackedFile::seekNormal() '%s' seek to pos="F_U64" failed: %s\n", _name, _pos * 8 + endianess_offset, strerror(errno)); exit(1); } errno = 0; size_t wordsread = read(_file, _bfr, sizeof(uint64) * _bfrmax); if (errno) { fprintf(stderr, "bitPackedFile::seekNormal() '%s' read of "F_U64" bytes failed': %s\n", _name, sizeof(uint64) * _bfrmax, strerror(errno)); exit(1); } // Flip all the words we just read, if needed // if (endianess_flipped) for (uint32 i=0; i> 6; if ((_pos <= np) && (np <= _pos + _bfrmax - 32)) { _bit = bitpos - (_pos << 6); stat_seekInside++; //fprintf(stderr, "SEEK INSIDE to _bit="F_U64"\n", _bit); return; } } if (_inCore) { fprintf(stderr, "bitPackedFile::seek()-- file '%s' is in core, but still needed to seek??\n", _name); exit(1); } stat_seekOutside++; flushDirty(); if (_isBzip2) seekBzip2(bitpos); else seekNormal(bitpos); _forceFirstLoad = false; //fprintf(stderr, "SEEK OUTSIDE to _pos="F_U64" _bit="F_U64"\n", _pos, _bit); } uint64 bitPackedFile::loadInCore(void) { struct stat sb; // Convert this disk-based, read/write bitPackedFile to memory-based read-only. flushDirty(); fstat(_file, &sb); // The extra 1024 words is to keep seek() from attempting to grab // the next block (there isn't a next block, we've got it all!) // when we're near the end of this block. We just make the block // a little bigger than it really is. delete [] _bfr; _bfrmax = sb.st_size / 8 + 1024; _bfr = new uint64 [_bfrmax]; _pos = 0; _bit = 0; // Tada! All we need to do now is load the block! _forceFirstLoad = true; seek(0); _inCore = true; return(_bfrmax * 8); } canu-1.3/src/AS_UTL/bitPackedFile.H000066400000000000000000000102471272335422300167270ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/bitPackedFile.H * * Modifications by: * * Brian P. Walenz from 2003-JAN-02 to 2003-OCT-20 * are Copyright 2003 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-MAR-29 to 2004-APR-21 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUL-12 to 2014-APR-11 * are Copyright 2005-2008,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef BITPACKEDFILE_H #define BITPACKEDFILE_H #include "AS_global.H" #include "bitOperations.H" #include "bitEncodings.H" #include "bitPacking.H" //#define WITH_BZIP2 #ifdef WITH_BZIP2 #include #endif class bitPackedFile { public: bitPackedFile(char const *name, uint64 offset=0, bool forceTruncate=false); ~bitPackedFile(); uint64 getBits(uint32 size); uint64 getNumber(void); void putBits(uint64 bits, uint32 size); void putNumber(uint64 val); uint64 tell(void) { return((_pos << 6) + _bit); }; void seek(uint64 pos); uint64 loadInCore(void); void showStats(FILE *f) { fprintf(f, "inside: "F_U64" outside: "F_U64"\n", stat_seekInside, stat_seekOutside); fflush(f); }; private: // Ensure that the buffer has enough space for any future // operation. This constant, currently 31 bytes, must be strictly // less than the constant used in deciding if seek() is moving // forward or backwards. // void sync(void) { if (((_bit >> 6) + 31) >= _bfrmax) seek((_pos << 6) + _bit); }; void flushDirty(void); void seekBzip2(uint64 bitpos); void seekNormal(uint64 bitpos); int _file; char *_name; #ifdef WITH_BZIP2 FILE *_bzFILE; int _bzerr; BZFILE *_bzfile; #endif uint64 _bfrmax; // Number of words in the buffer uint64 *_bfr; // A chunk of the bitPackedFile in core uint64 _pos; // The location this chunk is from (in words) uint64 _bit; // The bit position we are modifying relative to _pos bool _inCore; bool _bfrDirty; bool _forceFirstLoad; bool _isReadOnly; bool _isBzip2; // For collecting statistics on our usage // uint64 stat_seekInside; uint64 stat_seekOutside; uint64 stat_dirtyFlushes; // For converting between hardware of different endianess. // uint64 file_offset; uint64 endianess_offset; bool endianess_flipped; }; inline uint64 bitPackedFile::getBits(uint32 siz) { sync(); uint64 ret = getDecodedValue(_bfr, _bit, siz); _bit += siz; return(ret); } inline uint64 bitPackedFile::getNumber(void) { sync(); uint64 siz = 0; uint64 ret = getFibonacciEncodedNumber(_bfr, _bit, &siz); _bit += siz; return(ret); } inline void bitPackedFile::putBits(uint64 bits, uint32 siz) { assert(_isReadOnly == false); sync(); setDecodedValue(_bfr, _bit, siz, bits); _bit += siz; _bfrDirty = true; } inline void bitPackedFile::putNumber(uint64 val) { assert(_isReadOnly == false); sync(); uint64 siz = 0; setFibonacciEncodedNumber(_bfr, _bit, &siz, val); _bit += siz; _bfrDirty = true; } #endif // BITPACKEDFILE_H canu-1.3/src/AS_UTL/bitPacking.H000066400000000000000000000342101272335422300163100ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/bitPacking.h * * Modifications by: * * Brian P. Walenz from 2004-APR-27 to 2004-JUN-23 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2004-MAY-06 * are Copyright 2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2007-DEC-05 to 2014-APR-11 * are Copyright 2007-2008,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef BRI_BITPACKING_H #define BRI_BITPACKING_H #include #include // Routines used for stuffing bits into a word array. // Define this to enable testing that the width of the data element // is greater than zero. The uint64MASK() macro (bri.h) does not // generate a mask for 0. Compiler warnings are issued, because you // shouldn't use this in production code. // //#define CHECK_WIDTH // As CHECK_WIDTH is kind of expensive, we'll warn. #ifdef CHECK_WIDTH #warning CHECK_WIDTH is EXPENSIVE #endif // Returns 'siz' bits from the stream based at 'ptr' and currently at // location 'pos'. The position of the stream is not changed. // // Retrieves a collection of values; the number of bits advanced in // the stream is returned. // // Copies the lowest 'siz' bits in 'val' to the stream based at 'ptr' // and currently at 'pos'. The position of the stream is not // changed. // // Sets a collection of values; the number of bits advanced in the // stream is returned. // uint64 getDecodedValue (uint64 *ptr, uint64 pos, uint64 siz); uint64 getDecodedValues(uint64 *ptr, uint64 pos, uint64 num, uint64 *sizs, uint64 *vals); void setDecodedValue (uint64 *ptr, uint64 pos, uint64 siz, uint64 val); uint64 setDecodedValues(uint64 *ptr, uint64 pos, uint64 num, uint64 *sizs, uint64 *vals); // Like getDecodedValue() but will pre/post increment/decrement the // value stored in the stream before in addition to returning the // value. // // preIncrementDecodedValue(ptr, pos, siz) === x = getDecodedValue(ptr, pos, siz) + 1; // setDecodedValue(ptr, pos, siz, x); // // preDecrementDecodedValue(ptr, pos, siz) === x = getDecodedValue(ptr, pos, siz) - 1; // setDecodedValue(ptr, pos, siz, x); // // postIncrementDecodedValue(ptr, pos, siz) === x = getDecodedValue(ptr, pos, siz); // setDecodedValue(ptr, pos, siz, x + 1); // // postDecrementDecodedValue(ptr, pos, siz) === x = getDecodedValue(ptr, pos, siz); // setDecodedValue(ptr, pos, siz, x - 1); // uint64 preIncrementDecodedValue(uint64 *ptr, uint64 pos, uint64 siz); uint64 preDecrementDecodedValue(uint64 *ptr, uint64 pos, uint64 siz); uint64 postIncrementDecodedValue(uint64 *ptr, uint64 pos, uint64 siz); uint64 postDecrementDecodedValue(uint64 *ptr, uint64 pos, uint64 siz); // N.B. - I assume the bits in words are big-endian, which is // backwards from the way we shift things around. // // I define the "addresses" of bits in two consectuve words as // [0123][0123]. When adding words to the bit array, they're added // from left to right: // // setDecodedValue(bitstream, %0abc, 3) // setDecodedValue(bitstream, %0def, 3) // // results in [abcd][ef00] // // But when shifting things around, we typically do it from the right // side, since that is where the machine places numbers. // // A picture or two might help. // // // |----b1-----| // |-bit-||-sz-| // XXXXXX // [0---------------63] // ^ // pos // // // If the bits span two words, it'll look like this; b1 is smaller // than siz, and we update bit to be the "uncovered" piece of XXX // (all the stuff in word2). The first word is masked, then those // bits are shifted onto the result in the correct place. The second // word has the correct bits shifted to the right, then those are // appended to the result. // // |b1-| // |-----bit-----||---sz---| // XXXXXXXXXX // [0------------word1][0-------------word2] // ^ // pos // inline uint64 getDecodedValue(uint64 *ptr, uint64 pos, uint64 siz) { uint64 wrd = (pos >> 6) & 0x0000cfffffffffffllu; //PREFETCH(ptr + wrd); makes it worse uint64 bit = (pos ) & 0x000000000000003fllu; uint64 b1 = 64 - bit; uint64 ret = 0; #ifdef CHECK_WIDTH if (siz == 0) { fprintf(stderr, "ERROR: getDecodedValue() called with zero size!\n"); abort(); } if (siz > 64) { fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("uint64FMT")!\n", siz); abort(); } #endif if (b1 >= siz) { ret = ptr[wrd] >> (b1 - siz); } else { bit = siz - b1; ret = (ptr[wrd] & uint64MASK(b1)) << bit; wrd++; ret |= (ptr[wrd] >> (64 - bit)) & uint64MASK(bit); } ret &= uint64MASK(siz); return(ret); } inline void setDecodedValue(uint64 *ptr, uint64 pos, uint64 siz, uint64 val) { uint64 wrd = (pos >> 6) & 0x0000cfffffffffffllu; uint64 bit = (pos ) & 0x000000000000003fllu; uint64 b1 = 64 - bit; #ifdef CHECK_WIDTH if (siz == 0) { fprintf(stderr, "ERROR: setDecodedValue() called with zero size!\n"); abort(); } if (siz > 64) { fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("uint64FMT")!\n", siz); abort(); } #endif val &= uint64MASK(siz); if (b1 >= siz) { ptr[wrd] &= ~( uint64MASK(siz) << (b1 - siz) ); ptr[wrd] |= val << (b1 - siz); } else { bit = siz - b1; ptr[wrd] &= ~uint64MASK(b1); ptr[wrd] |= (val & (uint64MASK(b1) << (bit))) >> (bit); wrd++; ptr[wrd] &= ~(uint64MASK(bit) << (64 - bit)); ptr[wrd] |= (val & (uint64MASK(bit))) << (64 - bit); } } inline uint64 getDecodedValues(uint64 *ptr, uint64 pos, uint64 num, uint64 *sizs, uint64 *vals) { // compute the location of the start of the encoded words, then // just walk through to get the remaining words. uint64 wrd = (pos >> 6) & 0x0000cfffffffffffllu; //PREFETCH(ptr + wrd); makes it worse uint64 bit = (pos ) & 0x000000000000003fllu; uint64 b1 = 0; for (uint64 i=0; i 64) { fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("uint64FMT")!\n", siz); abort(); } #endif if (b1 >= sizs[i]) { //fprintf(stderr, "get-single pos=%d b1=%d bit=%d wrd=%d\n", pos, b1, bit, wrd); vals[i] = ptr[wrd] >> (b1 - sizs[i]); bit += sizs[i]; } else { //fprintf(stderr, "get-double pos=%d b1=%d bit=%d wrd=%d bitafter=%d\n", pos, b1, bit, wrd, sizs[i]-b1); bit = sizs[i] - b1; vals[i] = (ptr[wrd] & uint64MASK(b1)) << bit; wrd++; vals[i] |= (ptr[wrd] >> (64 - bit)) & uint64MASK(bit); } if (bit == 64) { wrd++; bit = 0; } assert(bit < 64); vals[i] &= uint64MASK(sizs[i]); pos += sizs[i]; } return(pos); } inline uint64 setDecodedValues(uint64 *ptr, uint64 pos, uint64 num, uint64 *sizs, uint64 *vals) { uint64 wrd = (pos >> 6) & 0x0000cfffffffffffllu; uint64 bit = (pos ) & 0x000000000000003fllu; uint64 b1 = 0; for (uint64 i=0; i 64) { fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("uint64FMT")!\n", siz); abort(); } #endif if (b1 >= sizs[i]) { //fprintf(stderr, "set-single pos=%d b1=%d bit=%d wrd=%d\n", pos, b1, bit, wrd); ptr[wrd] &= ~( uint64MASK(sizs[i]) << (b1 - sizs[i]) ); ptr[wrd] |= vals[i] << (b1 - sizs[i]); bit += sizs[i]; } else { //fprintf(stderr, "set-double pos=%d b1=%d bit=%d wrd=%d bitafter=%d\n", pos, b1, bit, wrd, sizs[i]-b1); bit = sizs[i] - b1; ptr[wrd] &= ~uint64MASK(b1); ptr[wrd] |= (vals[i] & (uint64MASK(b1) << (bit))) >> (bit); wrd++; ptr[wrd] &= ~(uint64MASK(bit) << (64 - bit)); ptr[wrd] |= (vals[i] & (uint64MASK(bit))) << (64 - bit); } if (bit == 64) { wrd++; bit = 0; } assert(bit < 64); pos += sizs[i]; } return(pos); } inline uint64 preIncrementDecodedValue(uint64 *ptr, uint64 pos, uint64 siz) { uint64 wrd = (pos >> 6) & 0x0000cfffffffffffllu; uint64 bit = (pos ) & 0x000000000000003fllu; uint64 b1 = 64 - bit; uint64 ret = 0; #ifdef CHECK_WIDTH if (siz == 0) { fprintf(stderr, "ERROR: preIncrementDecodedValue() called with zero size!\n"); abort(); } if (siz > 64) { fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("uint64FMT")!\n", siz); abort(); } #endif if (b1 >= siz) { ret = ptr[wrd] >> (b1 - siz); ret++; ret &= uint64MASK(siz); ptr[wrd] &= ~( uint64MASK(siz) << (b1 - siz) ); ptr[wrd] |= ret << (b1 - siz); } else { bit = siz - b1; ret = (ptr[wrd] & uint64MASK(b1)) << bit; ret |= (ptr[wrd+1] >> (64 - bit)) & uint64MASK(bit); ret++; ret &= uint64MASK(siz); ptr[wrd] &= ~uint64MASK(b1); ptr[wrd] |= (ret & (uint64MASK(b1) << (bit))) >> (bit); wrd++; ptr[wrd] &= ~(uint64MASK(bit) << (64 - bit)); ptr[wrd] |= (ret & (uint64MASK(bit))) << (64 - bit); } return(ret); } inline uint64 preDecrementDecodedValue(uint64 *ptr, uint64 pos, uint64 siz) { uint64 wrd = (pos >> 6) & 0x0000cfffffffffffllu; uint64 bit = (pos ) & 0x000000000000003fllu; uint64 b1 = 64 - bit; uint64 ret = 0; #ifdef CHECK_WIDTH if (siz == 0) { fprintf(stderr, "ERROR: preDecrementDecodedValue() called with zero size!\n"); abort(); } if (siz > 64) { fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("uint64FMT")!\n", siz); abort(); } #endif if (b1 >= siz) { ret = ptr[wrd] >> (b1 - siz); ret--; ret &= uint64MASK(siz); ptr[wrd] &= ~( uint64MASK(siz) << (b1 - siz) ); ptr[wrd] |= ret << (b1 - siz); } else { bit = siz - b1; ret = (ptr[wrd] & uint64MASK(b1)) << bit; ret |= (ptr[wrd+1] >> (64 - bit)) & uint64MASK(bit); ret--; ret &= uint64MASK(siz); ptr[wrd] &= ~uint64MASK(b1); ptr[wrd] |= (ret & (uint64MASK(b1) << (bit))) >> (bit); wrd++; ptr[wrd] &= ~(uint64MASK(bit) << (64 - bit)); ptr[wrd] |= (ret & (uint64MASK(bit))) << (64 - bit); } return(ret); } inline uint64 postIncrementDecodedValue(uint64 *ptr, uint64 pos, uint64 siz) { uint64 wrd = (pos >> 6) & 0x0000cfffffffffffllu; uint64 bit = (pos ) & 0x000000000000003fllu; uint64 b1 = 64 - bit; uint64 ret = 0; #ifdef CHECK_WIDTH if (siz == 0) { fprintf(stderr, "ERROR: postIncrementDecodedValue() called with zero size!\n"); abort(); } if (siz > 64) { fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("uint64FMT")!\n", siz); abort(); } #endif if (b1 >= siz) { ret = ptr[wrd] >> (b1 - siz); ret++; ret &= uint64MASK(siz); ptr[wrd] &= ~( uint64MASK(siz) << (b1 - siz) ); ptr[wrd] |= ret << (b1 - siz); } else { bit = siz - b1; ret = (ptr[wrd] & uint64MASK(b1)) << bit; ret |= (ptr[wrd+1] >> (64 - bit)) & uint64MASK(bit); ret++; ret &= uint64MASK(siz); ptr[wrd] &= ~uint64MASK(b1); ptr[wrd] |= (ret & (uint64MASK(b1) << (bit))) >> (bit); wrd++; ptr[wrd] &= ~(uint64MASK(bit) << (64 - bit)); ptr[wrd] |= (ret & (uint64MASK(bit))) << (64 - bit); } ret--; ret &= uint64MASK(siz); return(ret); } inline uint64 postDecrementDecodedValue(uint64 *ptr, uint64 pos, uint64 siz) { uint64 wrd = (pos >> 6) & 0x0000cfffffffffffllu; uint64 bit = (pos ) & 0x000000000000003fllu; uint64 b1 = 64 - bit; uint64 ret = 0; #ifdef CHECK_WIDTH if (siz == 0) { fprintf(stderr, "ERROR: postDecrementDecodedValue() called with zero size!\n"); abort(); } if (siz > 64) { fprintf(stderr, "ERROR: getDecodedValue() called with huge size ("uint64FMT")!\n", siz); abort(); } #endif if (b1 >= siz) { ret = ptr[wrd] >> (b1 - siz); ret--; ret &= uint64MASK(siz); ptr[wrd] &= ~( uint64MASK(siz) << (b1 - siz) ); ptr[wrd] |= ret << (b1 - siz); } else { bit = siz - b1; ret = (ptr[wrd] & uint64MASK(b1)) << bit; ret |= (ptr[wrd+1] >> (64 - bit)) & uint64MASK(bit); ret--; ret &= uint64MASK(siz); ptr[wrd] &= ~uint64MASK(b1); ptr[wrd] |= (ret & (uint64MASK(b1) << (bit))) >> (bit); wrd++; ptr[wrd] &= ~(uint64MASK(bit) << (64 - bit)); ptr[wrd] |= (ret & (uint64MASK(bit))) << (64 - bit); } ret++; ret &= uint64MASK(siz); return(ret); } #endif // BRI_BITPACKING_H canu-1.3/src/AS_UTL/decodeBooleanString.C000066400000000000000000000026571272335422300201540ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-NOV-26 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ static bool decodeBoolean(char *feature, char *value) { bool ret = false; // Decodes a string with 0/1, false/true, no/yes into an integer flag. switch (value[0]) { case '0': case 'f': case 'F': case 'n': case 'N': ret = false; break; case '1': case 't': case 'T': case 'y': case 'Y': ret = true; break; default: fprintf(stderr, "decodeBoolean()-- feature '%s' has unknown boolean value '%s'\n", feature, value); break; } return(ret); } canu-1.3/src/AS_UTL/dnaAlphabets.C000066400000000000000000000223001272335422300166130ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "dnaAlphabets.H" dnaAlphabets alphabet; void dnaAlphabets::initTablesForACGTSpace(void) { int i, j; for (i=0; i<256; i++) { _whitespaceSymbol[i] = isspace(i) ? 1 : 0; _toLower[i] = tolower(i); _toUpper[i] = toupper(i); _letterToBits[i] = (unsigned char)0xff; _bitsToLetter[i] = (unsigned char)'?'; _bitsToColor[i] = (unsigned char)'?'; _complementSymbol[i] = (unsigned char)'?'; } for (i=0; i<128; i++) for (j=0; j<128; j++) _IUPACidentity[i][j] = 0; _letterToBits['a'] = _letterToBits['A'] = (unsigned char)0x00; _letterToBits['c'] = _letterToBits['C'] = (unsigned char)0x01; _letterToBits['g'] = _letterToBits['G'] = (unsigned char)0x02; _letterToBits['t'] = _letterToBits['T'] = (unsigned char)0x03; _letterToBits['0'] = (unsigned char)0x00; _letterToBits['1'] = (unsigned char)0x01; _letterToBits['2'] = (unsigned char)0x02; _letterToBits['3'] = (unsigned char)0x03; _bitsToLetter[0x00] = 'A'; _bitsToLetter[0x01] = 'C'; _bitsToLetter[0x02] = 'G'; _bitsToLetter[0x03] = 'T'; _bitsToColor[0x00] = '0'; _bitsToColor[0x01] = '1'; _bitsToColor[0x02] = '2'; _bitsToColor[0x03] = '3'; _complementSymbol['a'] = 't'; // a _complementSymbol['t'] = 'a'; // t _complementSymbol['u'] = 'a'; // u, Really, only for RNA _complementSymbol['g'] = 'c'; // g _complementSymbol['c'] = 'g'; // c _complementSymbol['y'] = 'r'; // c t _complementSymbol['r'] = 'y'; // a g _complementSymbol['s'] = 'w'; // g c _complementSymbol['w'] = 's'; // a t _complementSymbol['k'] = 'm'; // t/u g _complementSymbol['m'] = 'k'; // a c _complementSymbol['b'] = 'v'; // c g t _complementSymbol['d'] = 'h'; // a g t _complementSymbol['h'] = 'd'; // a c t _complementSymbol['v'] = 'b'; // a c g _complementSymbol['n'] = 'n'; // a c g t _complementSymbol['A'] = 'T'; // a _complementSymbol['T'] = 'A'; // t _complementSymbol['U'] = 'A'; // u, Really, only for RNA _complementSymbol['G'] = 'C'; // g _complementSymbol['C'] = 'G'; // c _complementSymbol['Y'] = 'R'; // c t _complementSymbol['R'] = 'Y'; // a g _complementSymbol['S'] = 'W'; // g c _complementSymbol['W'] = 'S'; // a t _complementSymbol['K'] = 'M'; // t/u g _complementSymbol['M'] = 'K'; // a c _complementSymbol['B'] = 'V'; // c g t _complementSymbol['D'] = 'H'; // a g t _complementSymbol['H'] = 'D'; // a c t _complementSymbol['V'] = 'B'; // a c g _complementSymbol['N'] = 'N'; // a c g t _complementSymbol['0'] = '0'; // ColorSpace is self-complementing _complementSymbol['1'] = '1'; _complementSymbol['2'] = '2'; _complementSymbol['3'] = '3'; _IUPACidentity['A']['A'] = 1; _IUPACidentity['C']['C'] = 1; _IUPACidentity['G']['G'] = 1; _IUPACidentity['T']['T'] = 1; _IUPACidentity['M']['A'] = 1; _IUPACidentity['M']['C'] = 1; _IUPACidentity['R']['A'] = 1; _IUPACidentity['R']['G'] = 1; _IUPACidentity['W']['A'] = 1; _IUPACidentity['W']['T'] = 1; _IUPACidentity['S']['C'] = 1; _IUPACidentity['S']['G'] = 1; _IUPACidentity['Y']['C'] = 1; _IUPACidentity['Y']['T'] = 1; _IUPACidentity['K']['G'] = 1; _IUPACidentity['K']['T'] = 1; _IUPACidentity['V']['A'] = 1; _IUPACidentity['V']['C'] = 1; _IUPACidentity['V']['G'] = 1; _IUPACidentity['H']['A'] = 1; _IUPACidentity['H']['C'] = 1; _IUPACidentity['H']['T'] = 1; _IUPACidentity['D']['A'] = 1; _IUPACidentity['D']['G'] = 1; _IUPACidentity['D']['T'] = 1; _IUPACidentity['B']['C'] = 1; _IUPACidentity['B']['G'] = 1; _IUPACidentity['B']['T'] = 1; _IUPACidentity['N']['A'] = 1; _IUPACidentity['N']['C'] = 1; _IUPACidentity['N']['G'] = 1; _IUPACidentity['N']['T'] = 1; _IUPACidentity['M']['M'] = 1; _IUPACidentity['R']['R'] = 1; _IUPACidentity['W']['W'] = 1; _IUPACidentity['S']['S'] = 1; _IUPACidentity['Y']['Y'] = 1; _IUPACidentity['K']['K'] = 1; _IUPACidentity['V']['V'] = 1; _IUPACidentity['H']['W'] = 1; _IUPACidentity['D']['D'] = 1; _IUPACidentity['B']['B'] = 1; _IUPACidentity['N']['N'] = 1; // Order isn't important // for (i='A'; i<'Z'; i++) for (j='A'; j<'Z'; j++) { if (_IUPACidentity[j][i]) _IUPACidentity[i][j] = 1; } // Case isn't important // for (i='A'; i<'Z'; i++) for (j='A'; j<'Z'; j++) { if (_IUPACidentity[j][i]) { _IUPACidentity[tolower(i)][tolower(j)] = 1; _IUPACidentity[tolower(i)][j ] = 1; _IUPACidentity[i ][tolower(j)] = 1; } } } void dnaAlphabets::initTablesForColorSpace(void) { int i, j; for (i=0; i<128; i++) for (j=0; j<128; j++) _baseToColor[i][j] = '.'; // Invalid // Supports transforming a base sequence to a color sequence. // Not sure how valid this is; treat every letter like it's a gap. // We then override ACGT to be the correct encoding. for (i='a'; i<='z'; i++) { _baseToColor['a'][i] = '4'; _baseToColor['c'][i] = '4'; _baseToColor['g'][i] = '4'; _baseToColor['t'][i] = '4'; _baseToColor['n'][i] = '4'; } for (i='a'; i<='z'; i++) { _baseToColor[i]['a'] = '0'; _baseToColor[i]['c'] = '1'; _baseToColor[i]['g'] = '2'; _baseToColor[i]['t'] = '3'; _baseToColor[i]['n'] = '4'; } _baseToColor['a']['a'] = '0'; _baseToColor['a']['c'] = '1'; _baseToColor['a']['g'] = '2'; _baseToColor['a']['t'] = '3'; _baseToColor['a']['n'] = '4'; _baseToColor['c']['a'] = '1'; _baseToColor['c']['c'] = '0'; _baseToColor['c']['g'] = '3'; _baseToColor['c']['t'] = '2'; _baseToColor['c']['n'] = '4'; _baseToColor['g']['a'] = '2'; _baseToColor['g']['c'] = '3'; _baseToColor['g']['g'] = '0'; _baseToColor['g']['t'] = '1'; _baseToColor['g']['n'] = '4'; _baseToColor['t']['a'] = '3'; _baseToColor['t']['c'] = '2'; _baseToColor['t']['g'] = '1'; _baseToColor['t']['t'] = '0'; _baseToColor['t']['n'] = '4'; for (i='a'; i<='z'; i++) for (j='a'; j<='z'; j++) { _baseToColor[toupper(i)][toupper(j)] = _baseToColor[i][j]; _baseToColor[tolower(i)][toupper(j)] = _baseToColor[i][j]; _baseToColor[toupper(i)][tolower(j)] = _baseToColor[i][j]; _baseToColor[tolower(i)][tolower(j)] = _baseToColor[i][j]; } // Supports composing colors _baseToColor['0']['0'] = '0'; _baseToColor['0']['1'] = '1'; _baseToColor['0']['2'] = '2'; _baseToColor['0']['3'] = '3'; _baseToColor['0']['4'] = '4'; _baseToColor['1']['0'] = '1'; _baseToColor['1']['1'] = '0'; _baseToColor['1']['2'] = '3'; _baseToColor['1']['3'] = '2'; _baseToColor['1']['4'] = '4'; _baseToColor['2']['0'] = '2'; _baseToColor['2']['1'] = '3'; _baseToColor['2']['2'] = '0'; _baseToColor['2']['3'] = '1'; _baseToColor['2']['4'] = '4'; _baseToColor['3']['0'] = '3'; _baseToColor['3']['1'] = '2'; _baseToColor['3']['2'] = '1'; _baseToColor['3']['3'] = '0'; _baseToColor['3']['4'] = '4'; // Supports transforming color sequence to base sequence. _baseToColor['a']['0'] = _baseToColor['A']['0'] = 'a'; _baseToColor['a']['1'] = _baseToColor['A']['1'] = 'c'; _baseToColor['a']['2'] = _baseToColor['A']['2'] = 'g'; _baseToColor['a']['3'] = _baseToColor['A']['3'] = 't'; _baseToColor['a']['4'] = _baseToColor['A']['4'] = 'n'; _baseToColor['c']['0'] = _baseToColor['C']['0'] = 'c'; _baseToColor['c']['1'] = _baseToColor['C']['1'] = 'a'; _baseToColor['c']['2'] = _baseToColor['C']['2'] = 't'; _baseToColor['c']['3'] = _baseToColor['C']['3'] = 'g'; _baseToColor['c']['4'] = _baseToColor['C']['4'] = 'n'; _baseToColor['g']['0'] = _baseToColor['G']['0'] = 'g'; _baseToColor['g']['1'] = _baseToColor['G']['1'] = 't'; _baseToColor['g']['2'] = _baseToColor['G']['2'] = 'a'; _baseToColor['g']['3'] = _baseToColor['G']['3'] = 'c'; _baseToColor['g']['4'] = _baseToColor['G']['4'] = 'n'; _baseToColor['t']['0'] = _baseToColor['T']['0'] = 't'; _baseToColor['t']['1'] = _baseToColor['T']['1'] = 'g'; _baseToColor['t']['2'] = _baseToColor['T']['2'] = 'c'; _baseToColor['t']['3'] = _baseToColor['T']['3'] = 'a'; _baseToColor['t']['4'] = _baseToColor['T']['4'] = 'n'; _baseToColor['n']['0'] = _baseToColor['N']['0'] = 'a'; _baseToColor['n']['1'] = _baseToColor['N']['1'] = 'c'; _baseToColor['n']['2'] = _baseToColor['N']['2'] = 'g'; _baseToColor['n']['3'] = _baseToColor['N']['3'] = 't'; _baseToColor['n']['4'] = _baseToColor['N']['4'] = 'n'; } canu-1.3/src/AS_UTL/dnaAlphabets.H000066400000000000000000000054571272335422300166360ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libbio/alphabet-generate.c * kmer/libbio/dnaAlphabets.H * * Modifications by: * * Brian P. Walenz from 2003-MAY-06 to 2003-OCT-21 * are Copyright 2003 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2004-MAY-28 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAR-05 to 2008-JUL-28 * are Copyright 2005-2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include class dnaAlphabets { public: dnaAlphabets() { initTablesForACGTSpace(); }; ~dnaAlphabets() { }; void initTablesForACGTSpace(void); void initTablesForColorSpace(void); bool isWhitespace(unsigned char x) { return(_whitespaceSymbol[x]); }; unsigned char toLower(unsigned char x) { return(_toLower[x]); }; unsigned char toUpper(unsigned char x) { return(_toUpper[x]); }; unsigned char letterToBits(unsigned char x) { return(_letterToBits[x]); }; unsigned char bitsToLetter(unsigned char x) { return(_bitsToLetter[x]); }; unsigned char complementSymbol(unsigned char x) { return(_complementSymbol[x]); }; bool validCompressedSymbol(unsigned char x) { return(_validCompressedSymbol[x]); }; private: unsigned char _whitespaceSymbol[256]; unsigned char _toLower[256]; unsigned char _toUpper[256]; unsigned char _letterToBits[256]; unsigned char _bitsToLetter[256]; unsigned char _bitsToColor[256]; unsigned char _complementSymbol[256]; unsigned char _validCompressedSymbol[256]; unsigned char _IUPACidentity[128][128]; unsigned char _baseToColor[128][128]; }; extern dnaAlphabets alphabet; canu-1.3/src/AS_UTL/findKeyAndValue.H000066400000000000000000000076641272335422300172630ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2014-NOV-21 to 2015-JAN-30 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-FEB-22 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ class KeyAndValue { public: KeyAndValue(char *line = NULL) { find(line); }; ~KeyAndValue() { } bool find(char *line); char *key(void) { return(key_); }; char *value(void) { return(val_); }; bool value_bool(void) { return((val_[0] == 't') || (val_[0] == 'T') || (val_[0] == '1')); }; int32 value_int32(void) { return(strtol (val_, NULL, 10)); }; int64 value_int64(void) { return(strtoll(val_, NULL, 10)); }; uint32 value_uint32(void) { return(strtoul (val_, NULL, 10)); }; uint64 value_uint64(void) { return(strtoull(val_, NULL, 10)); }; float value_float(void) { return(strtof(val_, NULL)); }; double value_double(void) { return(strtod(val_, NULL)); }; public: bool iscomment(char c) { return((c == '!') || (c == '#') || (c == 0)); }; bool isdelimiter(char c) { return((c == ':') || (c == '=') || isspace(c)); }; char *key_; char *val_; }; // Returns true if a key and value are found. line is modified. // bool KeyAndValue::find(char *line) { key_ = NULL; val_ = NULL; if (line == NULL) return(false); key_ = line; while (isspace(*key_) == true) // Spaces before the key key_++; if ((iscomment(*key_) == true) || // If we're at a comment right now, there is no key (*key_ == 0)) { // and we return failure. key_ = NULL; val_ = NULL; return(false); } val_ = key_; // We're at the key now while ((*val_ != 0) && (isdelimiter(*val_) == false)) // The key cannot contain a delimiter. val_++; *val_++ = 0; while (isdelimiter(*val_) == true) { // Spaces or delimiter after the key *val_ = 0; val_++; } if (*val_ == 0) // And there is no value, must be a filename. return(true); char *eol = val_; // We're at the value now // If quoted, all we need to do is find the other quote and stop. if ((*val_ == '"') || (*val_ == '\'')) { val_++; eol++; while (*eol != '"') // The value itself. eol++; // The value CAN contain delimiters and comment markers. *eol = 0; } // Otherwise, not quoted. Find the first comment marker (or eol) then backup to the first non-space. else { while (iscomment(*eol) == false) // The value MUST NOT contain delimiters or comment markers. eol++; // But it can contains spaces and other nasties. eol--; // Back up off the comment or eol. while (isspace(*eol) == true) // And keep backing up as long as we're a space. eol--; eol++; // Move past the last non-space, non-comment *eol = 0; // And terminate the value } return(true); } canu-1.3/src/AS_UTL/intervalList.H000066400000000000000000000432351272335422300167240ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/intervalList.H * * Modifications by: * * Brian P. Walenz from 2003-AUG-11 to 2003-AUG-13 * are Copyright 2003 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2004-APR-21 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUL-12 to 2014-APR-11 * are Copyright 2005-2007,2010,2013-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-07 to 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-30 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INTERVALLIST_H #define INTERVALLIST_H #include // iNum - lo, hi - coordinates of the interval // iVal - va - data stored at each interval // uint32 - ct - number of elements in this interval // - when merged, needs function that converts multiple iVal and a uint32 into a single iVal template class _intervalPair { public: iNum lo; iNum hi; uint32 ct; // Number of source intervals iVal va; // Value at this interval; default is 1 bool operator<(const _intervalPair &that) const { if (lo != that.lo) return(lo < that.lo); return(hi < that.hi); }; }; template class intervalDepthRegions { public: iNum pos; // Position of the change in depth iVal change; // The value associated with this object; added or subtracted from 'va'. bool open; // If true, the start of a new interval bool operator<(const intervalDepthRegions &that) const { if (pos != that.pos) return(pos < that.pos); return(open > that.open); }; }; template class intervalList { public: intervalList(uint32 initialSize=32) { _isSorted = true; _isMerged = true; _listLen = 0; _listMax = initialSize; _list = new _intervalPair [_listMax]; }; // Takes as input an unmerged intervalList, returns to a new set of intervals, one // for each 'depth'. Two intervals, (1,4) and (2,6) would return 'depths': // 1,2,1 bgn=1, end=2, depth=1 // 2,4,2 // 4,6,1 // intervalList(intervalList &IL) { _isSorted = false; _isMerged = false; _listLen = 0; _listMax = 0; _list = 0L; depth(IL); }; intervalList(intervalDepthRegions *id, uint32 idlen) { _isSorted = false; _isMerged = false; _listLen = 0; _listMax = 0; _list = 0L; #ifdef _GLIBCXX_PARALLEL // Don't use the parallel sort, not with the expense of starting threads. __gnu_sequential::sort(id, id + idlen); #else std::sort(id, id + idlen); #endif computeDepth(id, idlen); }; ~intervalList() { delete [] _list; }; intervalList &operator=(intervalList &src); void clear(void) { _isSorted = true; _isMerged = true; _listLen = 0; } void add(iNum position, iNum length, iVal value=0); void sort(void); void merge(iNum minOverlap=0); // Merge overlapping regions void merge(intervalList *IL); // Insert IL into this list void filterShort(iNum minLength); void intersect(intervalList &A, intervalList &B); uint32 overlapping(iNum lo, iNum hi, uint32 *&intervals, uint32 &intervalsLen, uint32 &intervalsMax); // Populates this intervalList with regions in A that are completely // contained in a region in B. // // Both A and B call merge(). // void contained(intervalList &A, intervalList &B); void invert(iNum lo, iNum hi); void depth(intervalList &A); uint32 numberOfIntervals(void) { return(_listLen); }; iNum sumOfLengths(void) { iNum len = 0; uint32 i = numberOfIntervals(); if (i > 0) while (i--) len += _list[i].hi - _list[i].lo; return(len); }; iNum &lo(uint32 i) { return(_list[i].lo); }; iNum &hi(uint32 i) { return(_list[i].hi); }; uint32 &count(uint32 i) { return(_list[i].ct); }; // Number of source intervals. uint32 &depth(uint32 i) { return(_list[i].ct); }; // Depth, if converted. iVal &value(uint32 i) { return(_list[i].va); }; // Value or sum of values. private: void computeDepth(intervalDepthRegions *id, uint32 idlen); bool _isSorted; bool _isMerged; uint32 _listMax; uint32 _listLen; _intervalPair *_list; }; template intervalList & intervalList::operator=(intervalList &src) { _isSorted = src._isSorted; _isMerged = src._isMerged; if (_listMax < src._listMax) { delete [] _list; _listMax = src._listMax; _list = new _intervalPair [_listMax]; } _listLen = src._listLen; memcpy(_list, src._list, _listLen * sizeof(_intervalPair)); return(*this); } template void intervalList::add(iNum position, iNum length, iVal val) { if (_listLen >= _listMax) { _listMax *= 2; _intervalPair *l = new _intervalPair [_listMax]; memcpy(l, _list, sizeof(_intervalPair) * _listLen); delete [] _list; _list = l; } _list[_listLen].lo = position; _list[_listLen].hi = position + length; _list[_listLen].ct = 1; _list[_listLen].va = val; // Could optimize, and search the list to see if these are false, // but that's rather expensive. _isSorted = false; _isMerged = false; _listLen++; } template void intervalList::sort(void) { if (_isSorted) return; if (_listLen > 1) #ifdef _GLIBCXX_PARALLEL // Don't use the parallel sort, not with the expense of starting threads. __gnu_sequential::sort(_list, _list + _listLen); #else std::sort(_list, _list + _listLen); #endif _isSorted = true; } template void intervalList::merge(iNum minOverlap) { uint32 thisInterval = 0; uint32 nextInterval = 1; if (_isMerged) return; sort(); while (nextInterval < _listLen) { if ((_list[thisInterval].lo == 0) && (_list[thisInterval].hi == 0)) { // Our interval is empty. Copy in the interval we are // examining and move to the next. // XXX This is probably useless, thisInterval should always be // valid. _list[thisInterval].lo = _list[nextInterval].lo; _list[thisInterval].hi = _list[nextInterval].hi; _list[thisInterval].ct = _list[nextInterval].ct; _list[thisInterval].va = _list[nextInterval].va; _list[nextInterval].lo = 0; _list[nextInterval].hi = 0; nextInterval++; } else { // This interval is valid. See if it overlaps with the next // interval. bool intersects = false; if ((_list[thisInterval].lo <= _list[nextInterval].lo) && (_list[nextInterval].hi <= _list[thisInterval].hi)) // next is contained in this intersects = true; if (_list[thisInterval].hi - minOverlap >= _list[nextInterval].lo) // next has thick overlap to this intersects = true; if (intersects) { // Got an intersection. // Merge nextInterval into thisInterval -- the hi range // is extended if the nextInterval range is larger. // if (_list[thisInterval].hi < _list[nextInterval].hi) _list[thisInterval].hi = _list[nextInterval].hi; _list[thisInterval].ct += _list[nextInterval].ct; _list[thisInterval].va += _list[nextInterval].va; // Clear the just merged nextInterval and move to the next one. // _list[nextInterval].lo = 0; _list[nextInterval].hi = 0; _list[nextInterval].ct = 0; _list[nextInterval].va = 0; nextInterval++; } else { // No intersection. Move along. Nothing to see here. // If there is a gap between the target and the examine (we // must have merged sometime in the past), copy examine to // the next target. thisInterval++; if (thisInterval != nextInterval) { _list[thisInterval].lo = _list[nextInterval].lo; _list[thisInterval].hi = _list[nextInterval].hi; _list[thisInterval].ct = _list[nextInterval].ct; _list[thisInterval].va = _list[nextInterval].va; } nextInterval++; } } } if (thisInterval+1 < _listLen) _listLen = thisInterval + 1; _isMerged = true; } template void intervalList::merge(intervalList *IL) { for (uint32 i=0; i_listLen; i++) add(IL->_list[i].lo, IL->_list[i].hi - IL->_list[i].lo); } template void intervalList::filterShort(iNum minLength) { uint32 fr = 0; uint32 to = 0; for (; fr < _listLen; fr++) { if (hi(fr) - lo(fr) <= minLength) continue; if (to != fr) _list[to] = _list[fr]; to++; } _listLen = to; } template void intervalList::invert(iNum invlo, iNum invhi) { merge(); // Create a new list to store the inversion // uint32 invLen = 0; uint32 invMax = _listLen + 2; _intervalPair *inv = new _intervalPair [invMax]; // Add the zeroth and only? if (_listLen == 0) { inv[invLen].lo = invlo; inv[invLen].hi = invhi; inv[invLen].ct = 1; inv[invLen].va = 0; invLen++; } // Add the first, then the pieces, then the last // else { if (invlo < _list[0].lo) { inv[invLen].lo = invlo; inv[invLen].hi = _list[0].lo; inv[invLen].ct = 1; inv[invLen].va = 0; invLen++; } for (uint32 i=1; i<_listLen; i++) { if (_list[i-1].hi < _list[i].lo) { inv[invLen].lo = _list[i-1].hi; inv[invLen].hi = _list[i].lo; inv[invLen].ct = 1; inv[invLen].va = 0; invLen++; } } if (_list[_listLen-1].hi < invhi) { inv[invLen].lo = _list[_listLen-1].hi; inv[invLen].hi = invhi; inv[invLen].ct = 1; inv[invLen].va = 0; invLen++; } } assert(invLen <= invMax); // Nuke the old list, swap in the new one delete [] _list; _list = inv; _listLen = invLen; _listMax = invMax; } template void intervalList::intersect(intervalList &A, intervalList &B) { A.merge(); B.merge(); uint32 ai = 0; uint32 bi = 0; while ((ai < A.numberOfIntervals()) && (bi < B.numberOfIntervals())) { uint32 al = A.lo(ai); uint32 ah = A.hi(ai); uint32 bl = B.lo(bi); uint32 bh = B.hi(bi); uint32 nl = 0; uint32 nh = 0; // If they intersect, make a new region // if ((al <= bl) && (bl < ah)) { nl = bl; nh = (ah < bh) ? ah : bh; } if ((bl <= al) && (al < bh)) { nl = al; nh = (ah < bh) ? ah : bh; } if (nl < nh) add(nl, nh - nl); // Advance the list with the earlier region. // if (ah < bh) { // A ends before B ai++; } else if (ah > bh) { // B ends before A bi++; } else { // Exactly the same ending! ai++; bi++; } } } // Populates an array with the intervals that are within the supplied interval. // // Naive implementation that is easy to verify (and that works on an unsorted list). // template uint32 intervalList::overlapping(iNum rangelo, iNum rangehi, uint32 *&intervals, uint32 &intervalsLen, uint32 &intervalsMax) { if (intervals == 0L) { intervalsMax = 256; intervals = new uint32 [intervalsMax]; } intervalsLen = 0; for (uint32 i=0; i<_listLen; i++) { if ((rangelo <= _list[i].hi) && (rangehi >= _list[i].lo)) { if (intervalsLen >= intervalsMax) { intervalsMax *= 2; uint32 *X = new uint32 [intervalsMax]; memcpy(X, intervals, sizeof(uint32) * intervalsLen); delete [] intervals; intervals = X; } intervals[intervalsLen++] = i; } } return(intervalsLen); } template void intervalList::contained(intervalList &A, intervalList &B) { A.merge(); B.merge(); uint32 ai = 0; uint32 bi = 0; while ((ai < A.numberOfIntervals()) && (bi < B.numberOfIntervals())) { uint32 al = A.lo(ai); uint32 ah = A.hi(ai); uint32 bl = B.lo(bi); uint32 bh = B.hi(bi); // If A is contained in B, make a new region. // if ((bl <= al) && (ah <= bh)) add(bl, bh - bl); #if 0 if ((al <= bl) && (bh <= ah)) add(al, ah - al); #endif // Advance the list with the earlier region. // if (ah < bh) { // A ends before B ai++; } else if (ah > bh) { // B ends before A bi++; } else { // Exactly the same ending! ai++; bi++; } } } template void intervalList::depth(intervalList &IL) { uint32 idlen = IL.numberOfIntervals() * 2; intervalDepthRegions *id = new intervalDepthRegions [idlen]; for (uint32 i=0; i void intervalList::computeDepth(intervalDepthRegions *id, uint32 idlen) { // No intervals input? No intervals output. _listLen = 0; if (idlen == 0) return; // Sort by coordinate. #ifdef _GLIBCXX_PARALLEL // Don't use the parallel sort, not with the expense of starting threads. __gnu_sequential::sort(id, id + idlen); #else std::sort(id, id + idlen); #endif // Allocate the (maximum possible) depth of coverage intervals if (_listMax < idlen) { delete [] _list; _listMax = idlen; _list = new _intervalPair [_listMax]; } // Init first interval. assert(id[0].open == true); _list[_listLen].lo = id[0].pos; _list[_listLen].hi = id[0].pos; _list[_listLen].ct = 1; _list[_listLen].va = id[0].change; uint32 nct; iVal nva; for (uint32 i=1; i 1) && (_list[_listLen-1].hi == _list[_listLen].lo) && (_list[_listLen-1].ct == _list[_listLen].ct) && (_list[_listLen-1].va == _list[_listLen].va)) { _list[_listLen-1].hi = _list[_listLen].hi; _listLen--; } #if 0 fprintf(stderr, "id[%2d] - list[%u] = lo=%u hi=%u ct=%u va=%d\n", i, _listLen, _list[_listLen].lo, _list[_listLen].hi, _list[_listLen].ct, _list[_listLen].va); #endif } assert(_listLen > 0); assert(_listLen <= _listMax); } #endif // INTERVALLIST_H canu-1.3/src/AS_UTL/intervalListTest.C000066400000000000000000000031701272335422300175510ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAR-10 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include #include typedef int8_t int8; typedef int16_t int16; typedef int32_t int32; typedef int64_t int64; typedef uint8_t uint8; typedef uint16_t uint16; typedef uint32_t uint32; typedef uint64_t uint64; #include "intervalList.H" int main(int argc, char **argv) { intervalList t1; t1.add(0, 10); t1.add(11,7); t1.add(20, 8); fprintf(stderr, "BEFORE:\n"); for (uint32 ii=0; ii KMER_WORDS * 32) fprintf(stderr, "kMer size too large; increase KMER_WORDS in libbio/kmer.H\n"), exit(1); _compressionLength = new uint32 [_merSize]; for (uint32 z=0; z<_merSize; z++) _compressionLength[z] = (cm) ? 0 : 1; if (tm) { _merStorage = new kMer [_templateLength * 2]; _merSizeValid = new uint32 [_templateLength]; for (uint32 i=0; i<2*_templateLength; i++) { _merStorage[i].setMerSize(_merSize); _merStorage[i].setMerSpan(_templateSpan); } // VERY IMPORTANT! Offset the valid length to adjust for the // template that every mer except the first is starting in the // middle of. // for (uint32 i=0; i<_templateLength; i++) _merSizeValid[i] = _merSize - i; } else { _merStorage = new kMer [2]; _merSizeValid = new uint32 [1]; _merStorage[0].setMerSize(_merSize); _merStorage[1].setMerSize(_merSize); _merSizeValid[0] = _merSizeValidZero; if (cm) { _merStorage[0].setMerSpan(0); _merStorage[1].setMerSpan(0); } } _fMer = _merStorage + 0; _rMer = _merStorage + 1; } kMerBuilder::~kMerBuilder() { delete [] _merSizeValid; delete [] _merStorage; delete [] _compressionLength; delete [] _template; } void kMerBuilder::clear(bool clearMer) { // Contiguous mers _merSizeValid[0] = _merSizeValidZero; // Compressed mers if (_compression) { _compressionIndex = 0; _compressionFirstIndex = 0; _compressionCurrentLength = 0; for (uint32 z=0; z<_merSize; z++) _compressionLength[z] = 0; _merStorage[0].setMerSpan(0); _merStorage[1].setMerSpan(0); } // Spaced mers if (_template) { for (uint32 i=0; i<2*_templateLength; i++) _merStorage[i].clear(); for (uint32 i=0; i<_templateLength; i++) _merSizeValid[i] = _merSize - i; _templatePos = 0; _templateMer = 0; _templateFirst = 1; } if (clearMer) { _fMer->clear(); _rMer->clear(); } } // // The addBase methods add a single base (cf - forward, cr - complemented) to // the mer. The return true if another base is needed to finish the mer, and // false if the mer is complete. // bool kMerBuilder::addBaseContiguous(uint64 cf, uint64 cr) { // Not a valid base, reset the mer to empty, and request more bases // (this is a slightly optimized version of clear()). if (cf & (unsigned char)0xfc) { clear(false); //_merSizeValid[0] = _merSizeValidZero; return(true); } // Add the base to both mers. *_fMer += cf; *_rMer -= cr; // If there aren't enough bases, request another one. if (_merSizeValid[0] + 1 < _merSizeValidIs) { _merSizeValid[0]++; return(true); } return(false); // Good! Don't need another letter. } bool kMerBuilder::addBaseCompressed(uint64 cf, uint64 cr) { // Not a valid base, reset the mer to empty, and request more bases. // if (cf & (unsigned char)0xfc) { clear(); return(true); } uint64 lb = theFMer().endOfMer(2); // Last base in the mer uint32 ms = theFMer().getMerSpan(); // Span BEFORE adding the mer if (_merSizeValid[0] <= _merSizeValidZero) lb = 9; // No valid last base (should probably be ~uint64ZERO, but that screws up diagnostic output) #ifdef DEBUGCOMP fprintf(stderr, "kMerBuilder::addBaseCompressed()-- lb="uint64FMT" cf="uint64FMT" ms="F_U32" ccl="F_U32" lvl="F_U32"\n", lb, cf, ms, _compressionCurrentLength, _compression); #endif // Always add one to the current length. When we started, it // was 0. This represents the length AFTER adding the base. // _compressionCurrentLength++; // If the lastbase is the same as the one we want to add (and // there IS a last base), and we've seen too many of these, // remember we've seen another letter in the run, and don't add // it. Request another letter. // if ((lb == cf) && // last is the same as this (_compressionCurrentLength > _compression)) { // run is already too big _compressionLength[_compressionIndex]++; _fMer->setMerSpan(ms + 1); _rMer->setMerSpan(ms + 1); #ifdef DEBUGCOMP fprintf(stderr, "kMerBuilder::addBaseCompressed()-- COMPRESSED currentIdx=%u first=%u", _compressionIndex, _compressionFirstIndex); for (uint32 x=0, y=_compressionFirstIndex; x<_merSize; x++) { fprintf(stderr, " %u(%d)", _compressionLength[y], y); y = (y + 1) % _merSize; } fprintf(stderr, "\n"); #endif return(true); } // Else, it's a new run (a different letter) or our run isn't // big enough to compress and we need to add the duplicate // letter. *_fMer += cf; *_rMer -= cr; // If this is a new letter, propagate the current length to the first letter in this run. That // way, when that letter is popped off the mer, we automagically update our span to include only // as many letters as are here. // // 01234567890 // // E.g. For sequence TATTTTTTAGT (that's 6 T's) with a mersize of 3 and compression 2, we'd have // mers with position: // // TATTTTTTAGT // #1 TAT position 0 (with lengths 1, 1, 1) uncompressed mer TAT // #2 ATT position 1 (with lengths 1, 1, 1) ATT // #3 TTA position 6 (with lengths 5, 1, 1) TTTTTTA // #4 TAG position 7 TAG // #5 AGT position 8 AGT // // In #2, because the length so far (1) is not >= the compression (2) we add a new base and // return. // // In #3, the current length is >= the compression, so we keep stuffing on T's and incrementing // the last length, stopping when we get the A. We now propagate the current length to the first // letter in the run. Special case, if the first letter in the run is the first letter in the // mer, we need to immediately update the span. #ifdef DEBUGCOMP fprintf(stderr, "kMerBuilder::addBaseCompressed()-- ADDNEWBASE currentIdx=%u first=%u", _compressionIndex, _compressionFirstIndex); for (uint32 x=0, y=_compressionFirstIndex; x<_merSize; x++) { fprintf(stderr, " %u(%d)", _compressionLength[y], y); y = (y + 1) % _merSize; } fprintf(stderr, "\n"); #endif // If we added a new letter, transfer the run-length count to the first letter in the previous // run. In the above example, when we built the run, the lengths are (1, 1, 5). That is, all // compression occurred on the last letter. When we shift off that first letter, we want to // remove as much of the run as possible. if (lb != cf) { if (_compressionFirstIndex != _compressionIndex) { _compressionLength[_compressionFirstIndex] += _compressionLength[_compressionIndex] - 1; _compressionLength[_compressionIndex] = 1; } _compressionFirstIndex = (_compressionIndex + 1) % _merSize; _compressionCurrentLength = 1; } _compressionIndex = (_compressionIndex + 1) % _merSize; ms -= _compressionLength[_compressionIndex]; // subtract the count for the letter we just shifted out #ifdef DEBUGCOMP fprintf(stderr, "kMerBuilder::addBaseCompressed()-- ADDNEWBASE shifted out at idx="F_U32" with "F_U32" positions; final span "F_U32"\n", _compressionIndex, _compressionLength[_compressionIndex], ms + 1); #endif _compressionLength[_compressionIndex] = 1; // one letter at this position _fMer->setMerSpan(ms + 1); _rMer->setMerSpan(ms + 1); // If there aren't enough bases, request another one. if (_merSizeValid[0] + 1 < _merSizeValidIs) { _merSizeValid[0]++; return(true); } return(false); // Good! Don't need another letter. } bool kMerBuilder::addBaseSpaced(uint64 cf, uint64 cr) { #ifdef DEBUGSPACE fprintf(stderr, "add %c templatePos=%u templateMer=%u\n", ch, _templatePos, _templateMer); #endif // We always advance the templatePos, unfortunately, we need to // use the current value throughout this function. If there // was a single return point, we could advance immediately // before returning. // uint32 tp = _templatePos; _templatePos = (_templatePos + 1) % _templateLength; // If we get an invalid letter, set all mers that would have // had a letter added to be broken. // if (cf & (unsigned char)0xfc) { for (uint32 m=0; m<_templateLength; m++) { uint32 tppos = (tp + _templateLength - m) % _templateLength; if (_template[tppos] == 1) { // Reset to 'zero', but make it skip over any remaining // positions in the current template. // _merSizeValid[m] = _merSizeValidZero + tppos - _templateLength + 1; #ifdef DEBUGSPACE fprintf(stderr, "-- invalid letter, reset mer %u to valid %u (mersizevalidzero=%u ttpos=%u templatelength=%u)\n", m, _merSizeValid[m], _merSizeValidZero, tppos, _templateLength); #endif } } if (_templateFirst == 0) _templateMer = (_templateMer + 1) % _templateLength; return(true); } // We have a valid letter, and add it to all the mers that the // template allows. // for (uint32 m=0; m<_templateLength; m++) { uint32 tppos = (tp + _templateLength - m) % _templateLength; if (_template[tppos] == 1) { _merStorage[2*m+0] += cf; _merStorage[2*m+1] -= cr; if (_merSizeValid[m] < _merSizeValidIs) _merSizeValid[m]++; #ifdef DEBUGSPACE fprintf(stderr, "push %c onto %d (at template %u) length = %u %s\n", ch, m, (tp + _templateLength - m) % _templateLength, _merSizeValid[m], (_merSizeValid[m] >= _merSizeValidIs) ? "complete" : ""); #endif } else if (_merSizeValid[m] <= _merSizeValidZero) { // The template doesn't want us to add a letter to the mer, // but we're adjusting for an aborted template, and we're // counting template positions (not just non-zero template // positions) when adjusting. // _merSizeValid[m]++; } } // If the current mer isn't long enough, we move to the next mer, // and request another letter. // if (_merSizeValid[_templateMer] < _merSizeValidIs) { if (_templateFirst == 0) _templateMer = (_templateMer + 1) % _templateLength; #ifdef DEBUGSPACE fprintf(stderr, "-- too short -- need more templateMer=%u templateFirst=%u\n", _templateMer, _templateFirst); #endif return(true); } // On startup, _templateMer is always 0 (the first mer) until // it is long enough to be a valid mer. Then, we clear // _templateFirst so that we can start advancing through mers. // Update the f and r pointers to the correct mers, advance our // template to the next, and terminate. // _fMer = _merStorage + 2 * _templateMer + 0; _rMer = _merStorage + 2 * _templateMer + 1; #ifdef DEBUGSPACE fprintf(stderr, "-- valid! (templateMer = %u)\n", _templateMer); #endif _templateFirst = 0; _templateMer = (_templateMer + 1) % _templateLength; return(false); // Good! Don't need another letter. } bool kMerBuilder::addBaseCompressedSpaced(uint64 UNUSED(cf), uint64 UNUSED(cr)) { fprintf(stderr, "kMerBuilder::addBaseCompressedSpace()-- Compressed and spaced mers not supported.\n"); exit(1); } canu-1.3/src/AS_UTL/kMer.H000066400000000000000000000230371272335422300151400ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libbio/kmer.H * * Modifications by: * * Brian P. Walenz from 2005-MAY-19 to 2014-APR-11 * are Copyright 2005-2008,2010-2011,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ // Copyright (c) 2005 J. Craig Venter Institute // Author: Brian Walenz // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received (LICENSE.txt) a copy of the GNU General Public // License along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // A 'simple' kMer datastructure. #ifndef BIO_KMER_H #define BIO_KMER_H // The maximum size of a mer. You get 32 bases per word, so // KMER_WORDS=4 will get you up to a 128-mer. // #define KMER_WORDS 1 #include "AS_global.H" #include "bitPackedFile.H" #include "dnaAlphabets.H" // The kmer class is in two mutually exclusive implementations. The first implementation // uses one word to store up to 32-mers. The second implementation uses multiple words // to store arbitrarily large kmers. // // Incomplete and out of date description of the methods: // // // Reverse all the words, reverse and complement the bases in // // each word, then shift right to align the edge. // // // virtual kMerInterface &reverseComplement(void) = 0; // virtual void clear(void); // // // Construct a mer by shifting bases onto the end: // // += shifts onto the right end // // -= shifts onto the left end // // // virtual void operator+=(uint64 x) = 0; // virtual void operator-=(uint64 x) = 0; // // // used by merStream at least // // // virtual void mask(bool) = 0; // // // Return the mer, as a 64-bit integer. If the mer is more than // // 32-bases long, then the left-most (the earliest, the start, etc) // // bases are used. // // // virtual operator uint64 () const = 0; // // // These are written/read in 5'endian, which isn't the most natural // // implementation. It's done this way to keep the sequence in // // order (e.g., the merStreamFile). Don't change the order. // // // // On the otherhand, the implementation (of write anyway) is // // basically the same as merToString(). // // // // Takes an optional number of BITS to write, pulled from the // // END of the mer. // // // virtual void writeToBitPackedFile(bitPackedFile *BPF, uint32 numBits=0) const = 0; // virtual void readFromBitPackedFile(bitPackedFile *BPF, uint32 numBits=0) = 0; // // // Returns a sub-mer from either the start (left end) or the end // // (right end) of the mer. The sub-mer must be at most 64 bits // // long. Yes, BITS. // // // // The start is difficult, because it can span multiple words. The // // end is always in the first word. // // // virtual uint64 startOfMer(uint32 bits) const = 0; // virtual uint64 endOfMer(uint32 bits) const = 0; // // // Set 'numbits' bits from (the end of) 'val' at bit position 'pos' // // in the mer. This is wildly low-level, but merylStreamReader // // needs it. // // // // The position is measured from the right end. // // (0, 8, X) would copy the bits 7 to 0 of X to bits 7 to 0 of the mer. // // // // Argh! Can't use set/getDecodedValue because that is doing things in the wrong order. // // // // Meryl // // // virtual uint64 getWord(uint32 wrd) const = 0; // { return(MERWORD(wrd)); }; // virtual void setWord(uint32 wrd, uint64 val) = 0; // { MERWORD(wrd) = val; }; // // // Show the mer as ascii // // // // Doesn't print the last full word, if it's on the word boundary // // // // We build the string right to left, print any partial word first, // // then print whole words until we run out of words to print. // // // virtual char *merToString(char *instr) const = 0; // #if KMER_WORDS == 1 #include "kMerTiny.H" typedef kMerTiny kMer; #else #include "kMerHuge.H" typedef kMerHuge kMer; #endif #undef DEBUGADDBASE #undef DEBUGCOMP #undef DEBUGSPACE class kMerBuilder { public: kMerBuilder(uint32 ms=0, uint32 cm=0, char *tm=0L); ~kMerBuilder(); // Clear all mer data, reset state to as just after construction. void clear(bool clearMer=true); // Returns true if we need another base to finish the mer. This // only occurs for compressed mers, if we are in a homopolymer run. // private: bool addBaseContiguous(uint64 cf, uint64 cr); bool addBaseCompressed(uint64 cf, uint64 cr); bool addBaseSpaced(uint64 cf, uint64 cr); bool addBaseCompressedSpaced(uint64 cf, uint64 cr); public: bool addBase(char ch) { uint64 cf = alphabet.letterToBits(ch); uint64 cr = alphabet.letterToBits(alphabet.complementSymbol(ch)); #ifdef DEBUGADDBASE fprintf(stderr, "addBase() %c\n", ch); #endif if (_style == 0) return(addBaseContiguous(cf, cr)); if (_style == 1) return(addBaseCompressed(cf, cr)); if (_style == 2) return(addBaseSpaced(cf, cr)); if (_style == 3) return(addBaseCompressedSpaced(cf, cr)); fprintf(stderr, "kMerBuilder::addBase()-- Invalid mer type %d.\n", _style); exit(1); return(false); } void mask(void) { _fMer->mask(true); _rMer->mask(false); }; kMer const &theFMer(void) { return(*_fMer); }; kMer const &theRMer(void) { return(*_rMer); }; kMer const &theCMer(void) { return((theFMer() < theRMer()) ? theFMer() : theRMer()); }; uint32 merSize(void) { return(_merSize); }; uint32 templateSpan(void) { return(_templateSpan); }; uint32 baseSpan(uint32 b) { return(_compressionLength[(_compressionIndex + 1 + b) % _merSize]);; }; private: // Style of builder we are uint32 _style; // Amount of the mer that has valid sequence. Sigh. I really needed a signed value here -- // where negative values mean that we first have to get to the end of the template that was // invalid, then we need to build a new mer. // // And, yes, just simply making it signed leads to all sortes of compiler warnings about // comparing signed and unsigned. And I've been here before, and those warnings just propate // endlessly. Just go away, Mr. Smartypants. // // Details: when building spaced seeds, if we hit an N in the middle of the template, we need to // invalidate the mer, but not start building a new mer until we exhaust the current template. // The example is template=1101. Suppose we hit an N at the second 1. We set the merSizeValid // to 0, and proceed. When we push on the base for the last 1 in the template, we'd increment // the merSizeValid. The first two 1's in the template would now create a mer big enough to be // valid, and we'd return it -- but now the template we're using is 0111. // // _merSizeValid is offset by _merSize (e.g., the true valid size is _merSizeValid - _merSize). // _merSizeValidIs is the size _merSizeValid needs to be in order for it to be valid. // Similarily, _merSizeValidZero is the value of zero (currently this is equal to _merSize). // uint32 _merSize; // Desired number of bases in the mer uint32 *_merSizeValid; // Actual number of bases in the mer uint32 _merSizeValidZero; // Definition of 'zero' bases in the mer uint32 _merSizeValidIs; // Definition of 'full' bases in the mer // An array of mers, we allocate all mers in one block kMer *_merStorage; // Pointer to the currently active mer kMer *_fMer; kMer *_rMer; // For compression uint32 _compression; uint32 _compressionIndex; // index into cL[] that is the last base in the mer uint32 _compressionFirstIndex; // index into cL[] that is the first base in a run uint32 *_compressionLength; // one per base uint32 _compressionCurrentLength; // For templates uint32 _templateSpan; // # of 0's and 1's in the template uint32 _templateLength; // length of the pattern in the template char *_template; // character string template uint32 _templatePos; // position we are building in the template uint32 _templateMer; // the mer we should output next uint32 _templateFirst; // if true, we're still building the initial mer }; #endif // BIO_KMER_H canu-1.3/src/AS_UTL/kMerHuge.H000066400000000000000000000300071272335422300157440ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libbio/kmerhuge.H * * Modifications by: * * Brian P. Walenz from 2007-SEP-13 to 2014-APR-11 * are Copyright 2007-2008,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #define MERWORD(N) _md[N] class kMerHuge { public: kMerHuge(uint32 ms=uint32ZERO) { setMerSize(ms); clear(); }; ~kMerHuge() { }; void setMerSize(uint32 ms); uint32 getMerSize(void) const { return(_merSize); }; void setMerSpan(uint32 ms) { _merSpan = ms; }; uint32 getMerSpan(void) const { return(_merSpan); }; kMerHuge &reverseComplement(void) { for (uint32 i=0, j=KMER_WORDS-1; i> 2) & 0x3333333333333333llu) | ((MERWORD(i) << 2) & 0xccccccccccccccccllu); MERWORD(i) = ((MERWORD(i) >> 4) & 0x0f0f0f0f0f0f0f0fllu) | ((MERWORD(i) << 4) & 0xf0f0f0f0f0f0f0f0llu); MERWORD(i) = ((MERWORD(i) >> 8) & 0x00ff00ff00ff00ffllu) | ((MERWORD(i) << 8) & 0xff00ff00ff00ff00llu); MERWORD(i) = ((MERWORD(i) >> 16) & 0x0000ffff0000ffffllu) | ((MERWORD(i) << 16) & 0xffff0000ffff0000llu); MERWORD(i) = ((MERWORD(i) >> 32) & 0x00000000ffffffffllu) | ((MERWORD(i) << 32) & 0xffffffff00000000llu); MERWORD(i) ^= 0xffffffffffffffffllu; } *this >>= KMER_WORDS * 64 - 2 * _merSize; return(*this); }; void clear(void) { for (uint32 i=0; i>=(uint32 x) { // thisWord, the word we shift bits into // thatWord, the word we shift bits out of // shift, the number of bits we shift // uint32 thisWord = 0; uint32 thatWord = x >> 6; uint32 shift = x & uint32MASK(6); // Do an initial word-size shift, to reduce the shift amount to // be less than wordsize. Fill any shifted-out words with zero. // if (thatWord) { while (thatWord < KMER_WORDS) MERWORD(thisWord++) = MERWORD(thatWord++); while (thisWord < KMER_WORDS) MERWORD(thisWord++) = 0; } // Do bit-size shift, of adjacent words // thisWord = 0; thatWord = 1; MERWORD(thisWord) >>= shift; while (thatWord < KMER_WORDS) { MERWORD(thisWord++) |= MERWORD(thatWord) << (64 - shift); MERWORD(thatWord++) >>= shift; } }; void operator<<=(uint32 x) { uint32 thisWord = KMER_WORDS; uint32 thatWord = KMER_WORDS - (x >> 6); uint32 shift = x & uint32MASK(6); if (thatWord != KMER_WORDS) { while (thatWord > 0) MERWORD(--thisWord) = MERWORD(--thatWord); while (thisWord > 0) MERWORD(--thisWord) = 0; } thisWord = KMER_WORDS; thatWord = KMER_WORDS - 1; MERWORD(thisWord-1) <<= shift; while (thatWord > 0) { --thisWord; --thatWord; MERWORD(thisWord) |= MERWORD(thatWord) >> (64 - shift); MERWORD(thatWord) <<= shift; } }; public: void operator+=(uint64 x) { *this <<= 2; assert((x & 0xfc) == 0); MERWORD(0) |= x & uint64NUMBER(0x3); }; void operator-=(uint64 x) { *this >>= 2; assert((x & 0xfc) == 0); MERWORD(_lastWord) |= (x & uint64NUMBER(0x3)) << _lastShift; }; void mask(bool full) { MERWORD(_maskWord) &= _mask; if (full) for (uint32 x=_maskWord+1; x r.MERWORD(i)) return(false); } return(false); }; bool operator>(kMerHuge const &r) const { for (uint32 i=KMER_WORDS; i--; ) { if (MERWORD(i) > r.MERWORD(i)) return(true); if (MERWORD(i) < r.MERWORD(i)) return(false); } return(false); }; bool operator<=(kMerHuge const &r) const { for (uint32 i=KMER_WORDS; i--; ) { if (MERWORD(i) < r.MERWORD(i)) return(true); if (MERWORD(i) > r.MERWORD(i)) return(false); } return(true); }; bool operator>=(kMerHuge const &r) const { for (uint32 i=KMER_WORDS; i--; ) { if (MERWORD(i) > r.MERWORD(i)) return(true); if (MERWORD(i) < r.MERWORD(i)) return(false); } return(true); }; int qsort_less(kMerHuge const &r) const { for (uint32 i=KMER_WORDS; i--; ) { if (MERWORD(i) < r.MERWORD(i)) return(-1); if (MERWORD(i) > r.MERWORD(i)) return(1); } return(0); }; public: operator uint64 () const {return(MERWORD(0));}; public: // these should work generically for both big and small void writeToBitPackedFile(bitPackedFile *BPF, uint32 numBits=0) const { if (numBits == 0) numBits = _merSize << 1; uint32 lastWord = numBits >> 6; if ((numBits & uint32MASK(6)) == 0) lastWord++; if (numBits & uint32MASK(6)) BPF->putBits(MERWORD(lastWord), numBits & uint32MASK(6)); while (lastWord > 0) { lastWord--; BPF->putBits(MERWORD(lastWord), 64); } }; void readFromBitPackedFile(bitPackedFile *BPF, uint32 numBits=0) { if (numBits == 0) numBits = _merSize << 1; uint32 lastWord = numBits >> 6; if ((numBits & uint32MASK(6)) == 0) lastWord++; if (numBits & uint32MASK(6)) MERWORD(lastWord) = BPF->getBits(numBits & uint32MASK(6)); while (lastWord > 0) { lastWord--; MERWORD(lastWord) = BPF->getBits(64); } }; public: // these should work generically for both big and small void setBits(uint32 pos, uint32 numbits, uint64 val) { uint32 wrd = pos >> 6; uint32 bit = pos & 0x3f; val &= uint64MASK(numbits); if (wrd >= KMER_WORDS) { fprintf(stderr, "kMer::setBits()-- ERROR: tried to set pos="F_U32" numbits="F_U32" larger than KMER_WORDS=%d\n", pos, numbits, KMER_WORDS), exit(1); } // If we have enough space in the word for the bits, replace // those bits in the word. Otherwise we need to split the value // into two pieces, and add to the end of the first word and the // start of the second. if (64 - bit >= numbits) { MERWORD(wrd) &= ~(uint64MASK(numbits) << bit); MERWORD(wrd) |= val << bit; } else { if (wrd+1 >= KMER_WORDS) { fprintf(stderr, "kMer::setBits()-- ERROR: tried to set pos="F_U32" numbits="F_U32" larger than KMER_WORDS=%d\n", pos, numbits, KMER_WORDS), exit(1); } uint32 b1 = 64 - bit; // bits in the first word uint32 b2 = numbits - b1; // bits in the second word MERWORD(wrd) &= ~(uint64MASK(b1) << bit); MERWORD(wrd) |= (val & uint64MASK(b1)) << bit; MERWORD(wrd+1) &= ~(uint64MASK(b2)); MERWORD(wrd+1) |= (val >> b1) & uint64MASK(b2); } }; uint64 getBits(uint32 pos, uint32 numbits) const { uint64 val = uint64ZERO; uint32 wrd = pos >> 6; uint32 bit = pos & 0x3f; if (wrd >= KMER_WORDS) { fprintf(stderr, "kMer::getBits()-- ERROR: tried to get pos="F_U32" numbits="F_U32" larger than KMER_WORDS=%d\n", pos, numbits, KMER_WORDS), exit(1); } if (64 - bit >= numbits) { val = MERWORD(wrd) >> bit; } else { if (wrd+1 >= KMER_WORDS) { fprintf(stderr, "kMer::getBits()-- ERROR: tried to get pos="F_U32" numbits="F_U32" larger than KMER_WORDS=%d\n", pos, numbits, KMER_WORDS), exit(1); } uint32 b1 = 64 - bit; // bits in the first word uint32 b2 = numbits - b1; // bits in the second word val = MERWORD(wrd) >> (64-b1); val |= (MERWORD(wrd+1) & uint64MASK(b2)) << b1; } val &= uint64MASK(numbits); return(val); }; public: // these should work generically for both big and small uint64 startOfMer(uint32 bits) const { return(getBits((_merSize << 1) - bits, bits)); }; uint64 endOfMer(uint32 bits) const { return(MERWORD(0) & uint64MASK(bits)); }; public: // these should work generically for both big and small uint64 getWord(uint32 wrd) const { return(MERWORD(wrd)); }; void setWord(uint32 wrd, uint64 val) { MERWORD(wrd) = val; }; public: char *merToString(char *instr) const; private: uint64 _md[KMER_WORDS]; // The _merSize is always the number of letters in the mer -- if we // are a spaced seed, it is the weight. // uint32 _merSize; uint32 _merSpan; // The mask is used to make sure the mer has only _merSize bases // set -- we can get more than that if we shift to the left. The // _maskWord is the word that we want to mask: // uint64 _mask; uint32 _maskWord; // For operator-=() (add a base to the left end) we need to know // what the last word is, and how far to shift the bits. // // _lastWord -- the last word that contains bases // _lastShift -- the amount we need to shift left to put bits 0 and 1 // into the last base uint32 _lastWord; uint32 _lastShift; }; inline void kMerHuge::setMerSize(uint32 ms) { _merSize = ms; _merSpan = ms; _lastWord = (2 * ms - 2) / 64; _lastShift = (2 * ms - 2) % 64; _mask = uint64ZERO; _maskWord = _merSize / 32; // Filled whole words with the mer, the mask is special-cased // to clear the whole next word, unless there is no whole next // word, then it does nothing on the last word. // // Otherwise, we can construct the mask as usual. // if ((_merSize % 32) == 0) { if (_maskWord >= KMER_WORDS) { _maskWord = KMER_WORDS - 1; _mask = ~uint64ZERO; } else { _maskWord = _merSize / 32; _mask = uint64ZERO; } } else { _mask = uint64MASK((_merSize % 32) << 1); } if (_maskWord >= KMER_WORDS) { fprintf(stderr, "kMer::setMerSize()-- ERROR! Desired merSize of "F_U32" larger than\n", _merSize); fprintf(stderr, " available storage space (KMER_WORDS=%d, max merSize %d).\n", KMER_WORDS, KMER_WORDS*32); exit(1); } } inline char * uint64ToMerString(uint32 ms, uint64 mer, char *str) { for (uint32 i=0; i> (2*i)) & 0x03]; str[ms] = 0; return(str); } inline char * kMerHuge::merToString(char *instr) const { uint32 lastWord = _merSize >> 5; char *str = instr; if ((_merSize & uint32MASK(6)) == 0) lastWord++; if (_merSize & uint32MASK(5)) { uint64ToMerString(_merSize & uint32MASK(5), MERWORD(lastWord), str); str += _merSize & uint32MASK(5); } while (lastWord > 0) { lastWord--; uint64ToMerString(32, MERWORD(lastWord), str); str += 32; } return(instr); }; canu-1.3/src/AS_UTL/kMerTiny.H000066400000000000000000000127621272335422300160070ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libbio/kmertiny.H * * Modifications by: * * Brian P. Walenz from 2007-SEP-13 to 2014-APR-11 * are Copyright 2007-2008,2010,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ class kMerTiny { public: kMerTiny(uint32 ms=uint32ZERO) { setMerSize(ms); clear(); }; ~kMerTiny() { }; void setMerSize(uint32 ms); uint32 getMerSize(void) const { return(_merSize); }; void setMerSpan(uint32 ms) { _merSpan = ms; }; uint32 getMerSpan(void) const { return(_merSpan); }; kMerTiny &reverseComplement(void) { // Reverse the mer _md = ((_md >> 2) & 0x3333333333333333llu) | ((_md << 2) & 0xccccccccccccccccllu); _md = ((_md >> 4) & 0x0f0f0f0f0f0f0f0fllu) | ((_md << 4) & 0xf0f0f0f0f0f0f0f0llu); _md = ((_md >> 8) & 0x00ff00ff00ff00ffllu) | ((_md << 8) & 0xff00ff00ff00ff00llu); _md = ((_md >> 16) & 0x0000ffff0000ffffllu) | ((_md << 16) & 0xffff0000ffff0000llu); _md = ((_md >> 32) & 0x00000000ffffffffllu) | ((_md << 32) & 0xffffffff00000000llu); // Complement the bases _md ^= 0xffffffffffffffffllu; // Shift and mask out the bases not in the mer _md >>= 64 - _merSize * 2; _md &= uint64MASK(_merSize * 2); return(*this); }; void clear(void) { _md = uint64ZERO; }; void smallest(void) { clear(); }; void largest(void) { clear(); reverseComplement(); }; private: void operator>>=(uint32 x) { _md >>= x; }; void operator<<=(uint32 x) { _md <<= x; }; public: void operator+=(uint64 x) { *this <<= 2; assert((x & 0xfc) == 0); _md |= x & uint64NUMBER(0x3); }; void operator-=(uint64 x) { *this >>= 2; assert((x & 0xfc) == 0); _md |= (x & uint64NUMBER(0x3)) << _lastShift; }; public: void mask(bool) { _md &= _mask; }; public: bool operator!=(kMerTiny const &r) const { return(_md != r._md); }; bool operator==(kMerTiny const &r) const { return(_md == r._md); }; bool operator< (kMerTiny const &r) const { return(_md < r._md); }; bool operator> (kMerTiny const &r) const { return(_md > r._md); }; bool operator<=(kMerTiny const &r) const { return(_md <= r._md); }; bool operator>=(kMerTiny const &r) const { return(_md >= r._md); }; int qsort_less(kMerTiny const &r) const { if (_md < r._md) return(-1); if (_md > r._md) return( 1); return(0); }; public: operator uint64 () const {return(_md);}; public: void writeToBitPackedFile(bitPackedFile *BPF, uint32 UNUSED(numBits)=0) const { BPF->putBits(_md, _merSize << 1); }; void readFromBitPackedFile(bitPackedFile *BPF, uint32 UNUSED(numBits)=0) { _md = BPF->getBits(_merSize << 1); }; public: void setBits(uint32 pos, uint32 numbits, uint64 val) { _md &= ~(uint64MASK(numbits) << pos); _md |= val << pos; }; uint64 getBits(uint32 pos, uint32 numbits) const { return((_md >> pos) & uint64MASK(numbits)); }; public: uint64 startOfMer(uint32 bits) const { return(getBits((_merSize << 1) - bits, bits)); }; uint64 endOfMer(uint32 bits) const { return(_md & uint64MASK(bits)); }; public: uint64 getWord(uint32 UNUSED(wrd)) const { return(_md); }; void setWord(uint32 UNUSED(wrd), uint64 val) { _md = val; }; public: char *merToString(char *instr) const; private: uint64 _md; // The _merSize is always the number of letters in the mer -- if we // are a spaced seed, it is the weight. // uint32 _merSize; uint32 _merSpan; // The mask is used to make sure the mer has only _merSize bases // set -- we can get more than that if we shift to the left. The // uint64 _mask; // For operator-=() (add a base to the left end) we need to know // what the last word is, and how far to shift the bits. // uint32 _lastShift; }; inline void kMerTiny::setMerSize(uint32 ms) { _merSize = ms; _merSpan = ms; _lastShift = (2 * ms - 2) % 64; _mask = uint64MASK(_merSize << 1); } inline char * kMerTiny::merToString(char *str) const { for (uint32 i=0; i<_merSize; i++) str[_merSize-i-1] = alphabet.bitsToLetter((_md >> (2*i)) & 0x03); str[_merSize] = 0; return(str); } #if 0 // This is not used anywhere, and is probably quite stale inline uint64 stringToMer(uint32 ms, char *str) { uint64 mer = 0L; for (uint32 i=0; i> (32-(n)))) // FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4. // Rotation is separate from addition to prevent recomputation. #define FF(a, b, c, d, x, s, ac) { \ (a) += F ((b), (c), (d)) + (x) + (uint32)(ac); \ (a) = ROTATE_LEFT ((a), (s)); \ (a) += (b); \ } #define GG(a, b, c, d, x, s, ac) { \ (a) += G ((b), (c), (d)) + (x) + (uint32)(ac); \ (a) = ROTATE_LEFT ((a), (s)); \ (a) += (b); \ } #define HH(a, b, c, d, x, s, ac) { \ (a) += H ((b), (c), (d)) + (x) + (uint32)(ac); \ (a) = ROTATE_LEFT ((a), (s)); \ (a) += (b); \ } #define II(a, b, c, d, x, s, ac) { \ (a) += I ((b), (c), (d)) + (x) + (uint32)(ac); \ (a) = ROTATE_LEFT ((a), (s)); \ (a) += (b); \ } // MD5 initialization. Begins an MD5 operation, writing a new context. // void MD5Init (MD5_CTX *context) { context->count[0] = context->count[1] = 0; // Load magic initialization constants. context->state[0] = 0x67452301; context->state[1] = 0xefcdab89; context->state[2] = 0x98badcfe; context->state[3] = 0x10325476; } // MD5 block update operation. Continues an MD5 message-digest // operation, processing another message block, and updating the // context. // void MD5Update (MD5_CTX *context, unsigned char const *input, size_t inputLen) { unsigned int i, index, partLen; // Compute number of bytes mod 64 index = (unsigned int)((context->count[0] >> 3) & 0x3F); // Update number of bits if ((context->count[0] += ((uint32)inputLen << 3)) < ((uint32)inputLen << 3)) context->count[1]++; context->count[1] += ((uint32)inputLen >> 29); partLen = 64 - index; // Transform as many times as possible. if (inputLen >= partLen) { memcpy(&context->buffer[index], input, partLen); MD5Transform(context->state, context->buffer); for (i = partLen; i + 63 < inputLen; i += 64) MD5Transform(context->state, &input[i]); index = 0; } else i = 0; // Buffer remaining input memcpy(&context->buffer[index], &input[i], inputLen-i); } // MD5 finalization. Ends an MD5 message-digest operation, writing the // the message digest and zeroizing the context. // void MD5Final (unsigned char digest[16], MD5_CTX *context) { unsigned char bits[8]; unsigned int index, padLen; // Save number of bits Encode (bits, context->count, 8); // Pad out to 56 mod 64. index = (unsigned int)((context->count[0] >> 3) & 0x3f); padLen = (index < 56) ? (56 - index) : (120 - index); MD5Update (context, PADDING, padLen); // Append length (before padding) MD5Update (context, bits, 8); // Store state in digest Encode (digest, context->state, 16); // Zeroize sensitive information. memset(context, 0, sizeof(*context)); } // MD5 basic transformation. Transforms state based on block. // static void MD5Transform(uint32 state[4], unsigned char const block[64]) { uint32 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; Decode(x, block, 64); // Round 1 FF (a, b, c, d, x[ 0], S11, 0xd76aa478); // 1 FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); // 2 FF (c, d, a, b, x[ 2], S13, 0x242070db); // 3 FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); // 4 FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); // 5 FF (d, a, b, c, x[ 5], S12, 0x4787c62a); // 6 FF (c, d, a, b, x[ 6], S13, 0xa8304613); // 7 FF (b, c, d, a, x[ 7], S14, 0xfd469501); // 8 FF (a, b, c, d, x[ 8], S11, 0x698098d8); // 9 FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); // 10 FF (c, d, a, b, x[10], S13, 0xffff5bb1); // 11 FF (b, c, d, a, x[11], S14, 0x895cd7be); // 12 FF (a, b, c, d, x[12], S11, 0x6b901122); // 13 FF (d, a, b, c, x[13], S12, 0xfd987193); // 14 FF (c, d, a, b, x[14], S13, 0xa679438e); // 15 FF (b, c, d, a, x[15], S14, 0x49b40821); // 16 // Round 2 GG (a, b, c, d, x[ 1], S21, 0xf61e2562); // 17 GG (d, a, b, c, x[ 6], S22, 0xc040b340); // 18 GG (c, d, a, b, x[11], S23, 0x265e5a51); // 19 GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); // 20 GG (a, b, c, d, x[ 5], S21, 0xd62f105d); // 21 GG (d, a, b, c, x[10], S22, 0x2441453); // 22 GG (c, d, a, b, x[15], S23, 0xd8a1e681); // 23 GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); // 24 GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); // 25 GG (d, a, b, c, x[14], S22, 0xc33707d6); // 26 GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); // 27 GG (b, c, d, a, x[ 8], S24, 0x455a14ed); // 28 GG (a, b, c, d, x[13], S21, 0xa9e3e905); // 29 GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); // 30 GG (c, d, a, b, x[ 7], S23, 0x676f02d9); // 31 GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); // 32 // Round 3 HH (a, b, c, d, x[ 5], S31, 0xfffa3942); // 33 HH (d, a, b, c, x[ 8], S32, 0x8771f681); // 34 HH (c, d, a, b, x[11], S33, 0x6d9d6122); // 35 HH (b, c, d, a, x[14], S34, 0xfde5380c); // 36 HH (a, b, c, d, x[ 1], S31, 0xa4beea44); // 37 HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); // 38 HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); // 39 HH (b, c, d, a, x[10], S34, 0xbebfbc70); // 40 HH (a, b, c, d, x[13], S31, 0x289b7ec6); // 41 HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); // 42 HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); // 43 HH (b, c, d, a, x[ 6], S34, 0x4881d05); // 44 HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); // 45 HH (d, a, b, c, x[12], S32, 0xe6db99e5); // 46 HH (c, d, a, b, x[15], S33, 0x1fa27cf8); // 47 HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); // 48 // Round 4 II (a, b, c, d, x[ 0], S41, 0xf4292244); // 49 II (d, a, b, c, x[ 7], S42, 0x432aff97); // 50 II (c, d, a, b, x[14], S43, 0xab9423a7); // 51 II (b, c, d, a, x[ 5], S44, 0xfc93a039); // 52 II (a, b, c, d, x[12], S41, 0x655b59c3); // 53 II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); // 54 II (c, d, a, b, x[10], S43, 0xffeff47d); // 55 II (b, c, d, a, x[ 1], S44, 0x85845dd1); // 56 II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); // 57 II (d, a, b, c, x[15], S42, 0xfe2ce6e0); // 58 II (c, d, a, b, x[ 6], S43, 0xa3014314); // 59 II (b, c, d, a, x[13], S44, 0x4e0811a1); // 60 II (a, b, c, d, x[ 4], S41, 0xf7537e82); // 61 II (d, a, b, c, x[11], S42, 0xbd3af235); // 62 II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); // 63 II (b, c, d, a, x[ 9], S44, 0xeb86d391); // 64 state[0] += a; state[1] += b; state[2] += c; state[3] += d; // Zeroize sensitive information. memset (x, 0, sizeof(x)); } // Encodes input (uint32) into output (unsigned char). Assumes len is // a multiple of 4. // static void Encode (unsigned char *output, uint32 *input, unsigned int len) { unsigned int i, j; for (i = 0, j = 0; j < len; i++, j += 4) { output[j] = (unsigned char)(input[i] & 0xff); output[j+1] = (unsigned char)((input[i] >> 8) & 0xff); output[j+2] = (unsigned char)((input[i] >> 16) & 0xff); output[j+3] = (unsigned char)((input[i] >> 24) & 0xff); } } // Decodes input (unsigned char) into output (uint32). Assumes len is // a multiple of 4. // static void Decode (uint32 *output, unsigned char const *input, unsigned int len) { unsigned int i, j; for (i = 0, j = 0; j < len; i++, j += 4) output[i] = ((uint32)input[j]) | (((uint32)input[j+1]) << 8) | (((uint32)input[j+2]) << 16) | (((uint32)input[j+3]) << 24); } //////////////////////////////////////////////////////////////////////////////// // // kmer glue functions // //////////////////////////////////////////////////////////////////////////////// int md5_compare(void const *a, void const *b) { md5_s const *A = (md5_s const *)a; md5_s const *B = (md5_s const *)b; if (A->a < B->a) return(-1); if (A->a > B->a) return(1); if (A->b < B->b) return(-1); if (A->b > B->b) return(1); return(0); } static const char *md5_letters = "0123456789abcdef"; char* md5_toascii(md5_s *m, char *s) { int i; for (i=0; i<16; i++) { s[15-i ] = md5_letters[(m->a >> 4*i) & 0x0f]; s[15-i+16] = md5_letters[(m->b >> 4*i) & 0x0f]; } s[32] = 0; return(s); } md5_s* md5_string(md5_s *m, char *s, uint32 l) { MD5_CTX ctx; unsigned char dig[16]; int i = 0; if (m == NULL) { errno = 0; m = new md5_s; if (errno) { fprintf(stderr, "md5_string()-- Can't allocate a md5_s.\n%s\n", strerror(errno)); exit(1); } } MD5Init(&ctx); MD5Update(&ctx, (unsigned char*)s, l); MD5Final(dig, &ctx); m->a = dig[0]; while (i<8) { m->a <<= 8; m->a |= dig[i++]; } m->b = dig[i++]; while (i<16) { m->b <<= 8; m->b |= dig[i++]; } return(m); } static md5_increment_s* md5_increment_initialize(void) { md5_increment_s *m; errno = 0; m = new md5_increment_s; if (errno) { fprintf(stderr, "md5_increment_*()-- Can't allocate a md5_increment_s.\n%s\n", strerror(errno)); exit(1); } m->context = new MD5_CTX; if (errno) { fprintf(stderr, "md5_increment_*()-- Can't allocate a md5 context.\n%s\n", strerror(errno)); exit(1); } MD5Init((MD5_CTX *)m->context); m->bufferPos = 0; return(m); } md5_increment_s* md5_increment_char(md5_increment_s *m, char s) { if (m == NULL) m = md5_increment_initialize(); m->buffer[m->bufferPos++] = s; if (m->bufferPos == MD5_BUFFER_SIZE) { MD5Update((MD5_CTX *)m->context, m->buffer, m->bufferPos); m->bufferPos = 0; } return(m); } md5_increment_s* md5_increment_block(md5_increment_s *m, char *s, uint32 l) { if (m == NULL) m = md5_increment_initialize(); MD5Update((MD5_CTX *)m->context, (unsigned char*)s, l); return(m); } void md5_increment_finalize(md5_increment_s *m) { MD5_CTX *ctx = (MD5_CTX *)m->context; unsigned char dig[16]; int i = 0; if (m->bufferPos > 0) { MD5Update((MD5_CTX *)m->context, m->buffer, m->bufferPos); m->bufferPos = 0; } MD5Final(dig, ctx); m->a = dig[0]; while (i<8) { m->a <<= 8; m->a |= dig[i++]; } m->b = dig[i++]; while (i<16) { m->b <<= 8; m->b |= dig[i++]; } m->context = 0L; delete ctx; } void md5_increment_destroy(md5_increment_s *m) { delete m; } canu-1.3/src/AS_UTL/md5.H000066400000000000000000000022211272335422300147170ustar00rootroot00000000000000#include "AS_global.H" typedef struct { uint64 a; uint64 b; uint32 i; // the iid, used in leaff uint32 pad; // keep us size compatible between 32- and 64-bit machines. } md5_s; #define MD5_BUFFER_SIZE 32*1024 typedef struct { uint64 a; uint64 b; void *context; int bufferPos; unsigned char buffer[MD5_BUFFER_SIZE]; } md5_increment_s; // Returns -1, 0, 1 depending on if a <, ==, > b. Suitable for // qsort(). // int md5_compare(void const *a, void const *b); // Converts an md5_s into a character string. s must be at least // 33 bytes long. // char *md5_toascii(md5_s *m, char *s); // Computes the md5 checksum on the string s. // md5_s *md5_string(md5_s *m, char *s, uint32 l); // Computes an md5 checksum piece by piece. // // If m is NULL, a new md5_increment_s is allocated and returned. // md5_increment_s *md5_increment_char(md5_increment_s *m, char s); md5_increment_s *md5_increment_block(md5_increment_s *m, char *s, uint32 l); void md5_increment_finalize(md5_increment_s *m); void md5_increment_destroy(md5_increment_s *m); canu-1.3/src/AS_UTL/memoryMappedFile.H000066400000000000000000000133111272335422300174730ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/memoryMappedFile.H * * Modifications by: * * Brian P. Walenz from 2012-FEB-16 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-26 to 2015-APR-21 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Sergey Koren beginning on 2015-DEC-15 * are a 'United States Government Work', and * are released in the public domain * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef AS_UTL_MEMORYMAPPEDFILE_H #define AS_UTL_MEMORYMAPPEDFILE_H #include "AS_global.H" #include "AS_UTL_fileIO.H" #include #include #include #include using namespace std; // The BSD's are able to map to an arbitrary position in the file, but the Linux's can only map to // multiples of pagesize. Thus, this class maps the whole file into address space, then passes out // pointers to pieces in it. This is slightly unfortunate, because array out-of-bounds will not be // caught. To be fair, on the BSD's the file is mapped to a length that is a multiple of pagesize, // so it would take a big out-of-bounds to fail. enum memoryMappedFileType { memoryMappedFile_readOnly = 0x00, memoryMappedFile_readWrite = 0x01 }; #ifndef MAP_POPULATE #define MAP_POPULATE 0 #endif class memoryMappedFile { public: memoryMappedFile(const char *name, memoryMappedFileType type = memoryMappedFile_readOnly) { strcpy(_name, name); _type = type; errno = 0; int fd = (_type == memoryMappedFile_readOnly) ? open(_name, O_RDONLY | O_LARGEFILE) : open(_name, O_RDWR | O_LARGEFILE); if (errno) fprintf(stderr, "memoryMappedFile()-- Couldn't open '%s' for mmap: %s\n", _name, strerror(errno)), exit(1); struct stat sb; fstat(fd, &sb); if (errno) fprintf(stderr, "memoryMappedFile()-- Couldn't stat '%s' for mmap: %s\n", _name, strerror(errno)), exit(1); _length = sb.st_size; _offset = 0; if (_length == 0) fprintf(stderr, "memoryMappedFile()-- File '%s' is empty, can't mmap.\n", _name), exit(1); // Map a region that allows reading, or reading and shared writing. One could add PROT_WRITE // to the readOnly, and modifications will be kept private (if MAP_SHARED is switched to // MAP_PRIVATE) to the process (and discarded at the end). // // FreeBSD supports MAP_NOCORE which will exclude the region from any core files generated. Linux does not support it. // // Linux supports MAP_NORESERVE which will not reserve swap space for the file. When reserved, a write is guaranteed to succeed. // // NOTA BENE!! Even though it is writable, it CANNOT be extended. _data = (_type == memoryMappedFile_readOnly) ? mmap(0L, _length, PROT_READ, MAP_FILE | MAP_PRIVATE | MAP_POPULATE, fd, 0) : mmap(0L, _length, PROT_READ | PROT_WRITE, MAP_FILE | MAP_SHARED, fd, 0); if (errno) fprintf(stderr, "memoryMappedFile()-- Couldn't mmap '%s' of length "F_SIZE_T": %s\n", _name, _length, strerror(errno)), exit(1); close(fd); //fprintf(stderr, "memoryMappedFile()-- File '%s' of length %lu is mapped.\n", _name, _length); }; ~memoryMappedFile() { if (_type == memoryMappedFile_readWrite) msync(_data, _length, MS_SYNC); munmap(_data, _length); }; // Return a pointer to position 'offset' in the file, and set the current position to 'offset + // length'. // // The length parameter is checked against the length of the file, and a fatal error occurs if // 'offset + length' exceeds the bounds of the file. // void *get(size_t offset, size_t length) { if (length == 0) length = _length - offset; if (offset + length > _length) fprintf(stderr, "memoryMappedFile()-- Requested "F_SIZE_T" bytes at position "F_SIZE_T" in file '%s', but only "F_SIZE_T" bytes in file.\n", length, offset, _name, _length), exit(1); _offset = offset + length; return((uint8 *)_data + offset); }; // Return a pointer to the current position in the file, and move the current position ahead by // 'length' bytes. // // get() (or get(0)) returns the current position without moving it. // void *get(size_t length=0) { return(get(_offset, length)); }; size_t length(void) { return(_length); }; memoryMappedFileType type(void) { return(_type); }; private: char _name[FILENAME_MAX]; memoryMappedFileType _type; size_t _length; // Length of the mapped file size_t _offset; // File pointer for reading void *_data; }; #endif // AS_UTL_MEMORYMAPPEDFILE_H canu-1.3/src/AS_UTL/memoryMappedFileTest.C000066400000000000000000000023421272335422300203300ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/memoryMappedFileTest.C * * Modifications by: * * Brian P. Walenz from 2014-NOV-26 to 2014-NOV-27 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "memoryMappedFile.H" int main(int argc, char **argv) { memoryMappedFileRW *write = new memoryMappedFileRW("testWrite"); for (uint32 ii=0; ii<1000000000; ii++) (*write)[ii] = ii; delete write; exit(0); } canu-1.3/src/AS_UTL/mt19937ar.C000066400000000000000000000112521272335422300156110ustar00rootroot00000000000000/* A C-program for MT19937, with initialization improved 2002/1/26. Coded by Takuji Nishimura and Makoto Matsumoto. Before using, initialize the state by using init_genrand(seed) or init_by_array(init_key, key_length). Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura, All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The names of its contributors may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Any feedback is very welcome. http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space) */ #include "mt19937ar.H" #include #include // initialize with a single seed void mtRandom::construct(uint32 s) { mt[0] = s; // See Knuth TAOCP Vol2. 3rd Ed. P.106 for multiplier. In the previous versions, MSBs of the seed // affect only MSBs of the array mt[]. // 2002/01/09 modified by Makoto Matsumoto for (mti=1; mti> 30)) + mti); mag01[0] = uint32ZERO; mag01[1] = MT_MATRIX_A; } /* initialize by an array with array-length */ /* init_key is the array for initializing keys */ /* key_length is its length */ /* slight change for C++, 2004/2/26 */ mtRandom::mtRandom(uint32 *init_key, uint32 key_length) { construct(19650218UL); int i = 1; int j = 0; int k = (MT_N > key_length ? MT_N : key_length); for (; k; k--) { mt[i] = (mt[i] ^ ((mt[i-1] ^ (mt[i-1] >> 30)) * 1664525UL)) + init_key[j] + j; /* non linear */ i++; j++; if (i >= MT_N) { mt[0] = mt[MT_N-1]; i=1; } if (j >= key_length) j=0; } for (k=MT_N-1; k; k--) { mt[i] = (mt[i] ^ ((mt[i-1] ^ (mt[i-1] >> 30)) * 1566083941UL)) - i; /* non linear */ i++; if (i>=MT_N) { mt[0] = mt[MT_N-1]; i=1; } } mt[0] = 0x80000000UL; /* MSB is 1; assuring non-zero initial array */ } /* generates a random number on [0,0xffffffff]-interval */ uint32 mtRandom::mtRandom32(void) { uint32 y = 0; // generate MT_N words at one time // if (mti >= MT_N) { int kk; for (kk=0; kk < MT_N - MT_M; kk++) { y = (mt[kk] & MT_UPPER_MASK) | (mt[kk+1] & MT_LOWER_MASK); mt[kk] = mt[kk + MT_M] ^ (y >> 1) ^ mag01[y & uint32ONE]; } for (; kk < MT_N-1; kk++) { y = (mt[kk] & MT_UPPER_MASK) | (mt[kk + 1] & MT_LOWER_MASK); mt[kk] = mt[kk + (MT_M - MT_N)] ^ (y >> 1) ^ mag01[y & uint32ONE]; } y = (mt[MT_N-1] & MT_UPPER_MASK) | (mt[0] & MT_LOWER_MASK); mt[MT_N-1] = mt[MT_M-1] ^ (y >> 1) ^ mag01[y & uint32ONE]; mti = 0; } y = mt[mti++]; /* Tempering */ y ^= (y >> 11); y ^= (y << 7) & 0x9d2c5680UL; y ^= (y << 15) & 0xefc60000UL; y ^= (y >> 18); return(y); } // generates a random number on gaussian distribution with 0 median and 1 std.dev. double mtRandom::mtRandomGaussian(void) { double x1=0, x2=0, w=0, y1=0, y2=0; // from http://www.taygeta.com/random/gaussian.html // // supposedly equivalent to // // y1 = sqrt(-2*ln(x1)) cos(2*pi*x2) // y2 = sqrt(-2*ln(x1)) sin(2*pi*x2) // // but stable when x1 close to zero do { x1 = 2.0 * mtRandomRealClosed() - 1.0; x2 = 2.0 * mtRandomRealClosed() - 1.0; w = x1 * x1 + x2 * x2; } while (w >= 1.0); w = sqrt( (-2.0 * log(w)) / w); y1 = x1 * w; y2 = x2 * w; return(y1); } canu-1.3/src/AS_UTL/mt19937ar.H000066400000000000000000000043421272335422300156200ustar00rootroot00000000000000#ifndef MT19937AR_H #define MT19937AR_H #include "AS_global.H" // Refactoring of // // A C-program for MT19937, with initialization improved 2002/1/26. // Coded by Takuji Nishimura and Makoto Matsumoto. // // to make it thread safe and (hopefully) more portable. // // 20040421, bpw // Refactored again, 20141208 for C++. static const uint32 MT_N = 624; // period parameters static const uint32 MT_M = 397; static const uint32 MT_MATRIX_A = 0x9908b0dfUL; // constant vector a static const uint32 MT_UPPER_MASK = 0x80000000UL; // most significant w-r bits static const uint32 MT_LOWER_MASK = 0x7fffffffUL; // least significant r bits class mtRandom { private: void construct(uint32 s); public: mtRandom() { construct(getpid() * time(NULL)); }; mtRandom(uint32 s) { construct(s); }; mtRandom(uint32 *init_key, uint32 key_length); ~mtRandom() { }; uint32 mtRandom32(void); uint64 mtRandom64(void) { return((((uint64)mtRandom32()) << 32) | (uint64)mtRandom32()); } // Real valued randomness // mtRandomRealOpen() -- on [0,1) real interval // mtRandomRealClosed() -- on [0,1] real interval // mrRandomRealOpen53() -- on [0,1) real interval, using 53 bits // // "These real versions are due to Isaku Wada, 2002/01/09 added" and were taken from // the mt19937ar.c distribution (but they had actual functions, not macros) // // They also had // random number in (0,1) as (mtRandom32() + 0.5) * (1.0 / 4294967296.0) // double mtRandomRealOpen(void) { return((double)mtRandom32() * (1.0 / 4294967296.0)); }; double mtRandomRealClosed(void) { return((double)mtRandom32() * (1.0 / 4294967295.0)); }; double mtRandomRealOpen53(void) { return(((mtRandom32() >> 5) * 67108864.0 + (mtRandom32() >> 6)) * (1.0 / 9007199254740992.0)); }; // returns a random number with gaussian distribution, mean of zero and std.dev. of 1 // double mtRandomGaussian(void); private: uint32 mt[MT_N]; // State vector array uint32 mti; // Ordinal of the first uninit'd element -- mti = N+1 -> elt N is uninit uint32 mag01[2]; // mag01[x] = x * MT_MATRIX_A for x=0,1 }; #endif // MT19937AR_H canu-1.3/src/AS_UTL/readBuffer.C000066400000000000000000000175201272335422300163020ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/readBuffer.C * * Modifications by: * * Brian P. Walenz on 2004-MAY-06 * are Copyright 2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-MAY-11 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-APR-01 to 2014-APR-11 * are Copyright 2005-2010,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-AUG-22 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "readBuffer.H" #include "memoryMappedFile.H" #if 0 #include #include #include #include #include #endif #include // If bufferMax is zero, then the file is accessed using memory // mapped I/O. Otherwise, a small buffer is used. // readBuffer::readBuffer(const char *filename, uint64 bufferMax) { _filename = 0L; _file = 0; _filePos = 0; _mmap = NULL; _stdin = false; _eof = false; _bufferPos = 0; _bufferLen = 0; _bufferMax = 0; _buffer = 0L; if (((filename == 0L) && (isatty(fileno(stdin)) == 0)) || ((filename != 0L) && (filename[0] == '-') && (filename[1] == 0))) { _filename = new char [32]; strcpy(_filename, "(stdin)"); _stdin = true; if (bufferMax == 0) bufferMax = 32 * 1024; } else if (filename == 0L) { fprintf(stderr, "readBuffer()-- no filename supplied, and I will not use the terminal for input.\n"), exit(1); } else { _filename = new char [strlen(filename) + 1]; strcpy(_filename, filename); } if (bufferMax == 0) { _mmap = new memoryMappedFile(_filename); _buffer = (char *)_mmap->get(0); } else { errno = 0; _file = (_stdin) ? fileno(stdin) : open(_filename, O_RDONLY | O_LARGEFILE); if (errno) fprintf(stderr, "readBuffer()-- couldn't open the file '%s': %s\n", _filename, strerror(errno)), exit(1); _bufferMax = bufferMax; _buffer = new char [_bufferMax]; } fillBuffer(); if (_bufferLen == 0) _eof = true; } readBuffer::readBuffer(FILE *file, uint64 bufferMax) { if (bufferMax == 0) fprintf(stderr, "readBuffer()-- WARNING: mmap() not supported in readBuffer(FILE *)\n"); _filename = new char [32]; _file = fileno(file); _filePos = 0; _mmap = NULL; _stdin = false; _eof = false; _bufferPos = 0; _bufferLen = 0; _bufferMax = (bufferMax == 0) ? 32 * 1024 : bufferMax; _buffer = new char [_bufferMax]; strcpy(_filename, "(hidden file)"); // Just be sure that we are at the start of the file. errno = 0; lseek(_file, 0, SEEK_SET); if ((errno) && (errno != ESPIPE)) fprintf(stderr, "readBuffer()-- '%s' couldn't seek to position 0: %s\n", _filename, strerror(errno)), exit(1); fillBuffer(); if (_bufferLen == 0) _eof = true; } readBuffer::~readBuffer() { delete [] _filename; if (_mmap) delete _mmap; else delete [] _buffer; if (_stdin == false) close(_file); } void readBuffer::fillBuffer(void) { // If there is still stuff in the buffer, no need to fill. if (_bufferPos < _bufferLen) return; // No more stuff in the buffer. But if mmap'd, ths means we're EOF. if (_mmap) { _eof = true; return; } _bufferPos = 0; _bufferLen = 0; again: errno = 0; _bufferLen = (uint64)::read(_file, _buffer, _bufferMax); if (errno == EAGAIN) goto again; if (errno) fprintf(stderr, "readBuffer::fillBuffer()-- only read "F_U64" bytes, couldn't read "F_U64" bytes from '%s': %s\n", _bufferLen, _bufferMax, _filename, strerror(errno)), exit(1); if (_bufferLen == 0) _eof = true; } void readBuffer::seek(uint64 pos) { if (_stdin == true) { if (_filePos < _bufferLen) { _filePos = 0; _bufferPos = 0; return; } else { fprintf(stderr, "readBuffer()-- seek() not available for file 'stdin'.\n"); exit(1); } return; } assert(_stdin == false); if (_mmap) { _bufferPos = pos; _filePos = pos; } else { errno = 0; lseek(_file, pos, SEEK_SET); if (errno) fprintf(stderr, "readBuffer()-- '%s' couldn't seek to position "F_U64": %s\n", _filename, pos, strerror(errno)), exit(1); _bufferLen = 0; _bufferPos = 0; _filePos = pos; fillBuffer(); } _eof = (_bufferPos >= _bufferLen); } uint64 readBuffer::read(void *buf, uint64 len) { char *bufchar = (char *)buf; // Handle the mmap'd file first. if (_mmap) { uint64 c = 0; while ((_bufferPos < _bufferLen) && (c < len)) { bufchar[c++] = _buffer[_bufferPos++]; _filePos++; } if (c == 0) _eof = true; return(c); } // Easy case; the next len bytes are already in the buffer; just // copy and move the position. if (_bufferLen - _bufferPos > len) { memcpy(bufchar, _buffer + _bufferPos, len); _bufferPos += len; fillBuffer(); _filePos += len; return(len); } // Existing buffer not big enough. Copy what's there, then finish // with a read. uint64 bCopied = 0; // Number of bytes copied into the buffer uint64 bRead = 0; // Number of bytes read into the buffer uint64 bAct = 0; // Number of bytes actually read from disk memcpy(bufchar, _buffer + _bufferPos, _bufferLen - _bufferPos); bCopied = _bufferLen - _bufferPos; _bufferPos = _bufferLen; while (bCopied + bRead < len) { errno = 0; bAct = (uint64)::read(_file, bufchar + bCopied + bRead, len - bCopied - bRead); if (errno) fprintf(stderr, "readBuffer()-- couldn't read "F_U64" bytes from '%s': n%s\n", len, _filename, strerror(errno)), exit(1); // If we hit EOF, return a short read if (bAct == 0) len = 0; bRead += bAct; } fillBuffer(); _filePos += bCopied + bRead; return(bCopied + bRead); } uint64 readBuffer::read(void *buf, uint64 maxlen, char stop) { char *bufchar = (char *)buf; uint64 c = 0; // We will copy up to 'maxlen'-1 bytes into 'buf', or stop at the first occurrence of 'stop'. // This will reserve space at the end of any string for a zero-terminating byte. maxlen--; if (_mmap) { // Handle the mmap'd file first. while ((_bufferPos < _bufferLen) && (c < maxlen)) { bufchar[c++] = _buffer[_bufferPos++]; if (bufchar[c-1] == stop) break; } if (_bufferPos >= _bufferLen) _eof = true; } else { // And the usual case. while ((_eof == false) && (c < maxlen)) { bufchar[c++] = _buffer[_bufferPos++]; if (_bufferPos >= _bufferLen) fillBuffer(); if (bufchar[c-1] == stop) break; } } bufchar[c] = 0; return(c); } canu-1.3/src/AS_UTL/readBuffer.H000066400000000000000000000066321272335422300163110ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/readBuffer.H * * Modifications by: * * Brian P. Walenz from 2003-APR-15 to 2004-JAN-06 * are Copyright 2003-2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-MAR-17 to 2004-MAY-24 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2006-JUN-24 to 2014-APR-11 * are Copyright 2006-2008,2010,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef READ_BUFFER_H #define READ_BUFFER_H #include "AS_global.H" #include "memoryMappedFile.H" class readBuffer { public: readBuffer(const char *filename, uint64 bufferMax = 32 * 1024); readBuffer(FILE *F, uint64 bufferMax = 32 * 1024); ~readBuffer(); bool eof(void) { return(_eof); }; char peek(void); char read(void); uint64 read(void *buf, uint64 len); uint64 read(void *buf, uint64 maxlen, char stop); void seek(uint64 pos); uint64 tell(void) { return(_filePos); }; const char *filename(void) { return(_filename); }; private: void fillBuffer(void); void init(int fileptr, const char *filename, uint64 bufferMax); char *_filename; int _file; uint64 _filePos; memoryMappedFile *_mmap; bool _stdin; bool _eof; // If bufferMax is zero, then we are using the mmapped interface, otherwise, // we are using a open()/read() and a small buffer. uint64 _bufferPos; uint64 _bufferLen; uint64 _bufferMax; char *_buffer; }; // Returns the next letter in the buffer, but DOES NOT advance past // it. Might have some wierd interaction with EOF -- if you peek() // and the next thing is eof , the _eof flag might get set. // inline char readBuffer::peek(void) { if ((_eof == false) && (_bufferPos >= _bufferLen)) fillBuffer(); if (_eof) return(0); return(_buffer[_bufferPos]); } // Returns the next letter in the buffer. Returns EOF (0) if there // is no next letter. // inline char readBuffer::read(void) { if ((_eof == false) && (_bufferPos >= _bufferLen)) fillBuffer(); if (_eof) return(0); _bufferPos++; _filePos++; return(_buffer[_bufferPos-1]); } #endif // READ_BUFFER_H canu-1.3/src/AS_UTL/speedCounter.C000066400000000000000000000056351272335422300167010ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/speedCounter.C * * Modifications by: * * Brian P. Walenz from 2006-OCT-22 to 2014-APR-11 * are Copyright 2006,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "speedCounter.H" const char* speedCounter::_spinr[4] = { "[|]", "[/]", "[-]", "[\\]" }; const char* speedCounter::_liner[19] = { "[- ]", "[-- ]", "[ -- ]", "[ -- ]", "[ -- ]", "[ -- ]", "[ -- ]", "[ -- ]", "[ -- ]", "[ --]", "[ -]", "[ --]", "[ -- ]", "[ -- ]", "[ -- ]", "[ -- ]", "[ -- ]", "[ -- ]", "[ -- ]" }; speedCounter::speedCounter(char const *fmt, double unit, uint64 freq, bool enabled) { _count = 0; _draws = 0; _unit = unit; _freq = freq; _startTime = getTime(); _fmt = fmt; _spin = false; _line = false; _enabled = enabled; // We use _draws instead of shifting _count just because it's // simpler, and both methods need another variable anyway. // Set all the bits below the hightest set in _freq -- // this allows us to do a super-fast test in tick(). // _freq |= _freq >> 1; _freq |= _freq >> 2; _freq |= _freq >> 4; _freq |= _freq >> 8; _freq |= _freq >> 16; _freq |= _freq >> 32; } speedCounter::~speedCounter() { finish(); } canu-1.3/src/AS_UTL/speedCounter.H000066400000000000000000000065161272335422300167050ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/speedCounter.H * * Modifications by: * * Brian P. Walenz from 2003-JAN-02 to 2003-MAY-06 * are Copyright 2003 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUL-12 to 2014-APR-11 * are Copyright 2005-2006,2012-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef SPEEDCOUNTER_H #define SPEEDCOUNTER_H #include "AS_global.H" #include "timeAndSize.H" class speedCounter { public: // fmt specifies the status format. An example: // " %8f [unit]things (%8.5f [unit]things/sec)\r" // speedCounter(char const *fmt, double unit, uint64 freq, bool enabled=true); ~speedCounter(); void enableSpinner(void) { _spin = true; }; void enableLiner(void) { _line = true; }; bool tick(void) { if (_enabled && ((++_count & _freq) == uint64ZERO)) { double v = _count / _unit; if (_spin) fputs(_spinr[_draws % 4], stderr); if (_line) fputs(_liner[_draws % 19], stderr); _draws++; fprintf(stderr, _fmt, v, v / (getTime() - _startTime)); fflush(stderr); return(true); } return(false); }; bool tick(uint64 increment) { if (_enabled == false) return(false); _count += increment; if ((_count & _freq) == uint64ZERO) { double v = _count / _unit; if (_spin) fputs(_spinr[_draws % 4], stderr); if (_line) fputs(_liner[_draws % 19], stderr); _draws++; fprintf(stderr, _fmt, v, v / (getTime() - _startTime)); fflush(stderr); return(true); } return(false); }; void finish(void) { if (_enabled && (_count >= _freq)) { double v = _count / _unit; if (_spin) fputs(_spinr[_draws % 4], stderr); if (_line) fputs(_liner[_draws % 19], stderr); fprintf(stderr, _fmt, v, v / (getTime() - _startTime)); fprintf(stderr, "\n"); fflush(stderr); } _count = 0; }; private: static const char *_spinr[4]; static const char *_liner[19]; uint64 _count; uint64 _draws; double _unit; uint64 _freq; double _startTime; char const *_fmt; bool _spin; bool _line; bool _enabled; }; #endif // SPEEDCOUNTER_H canu-1.3/src/AS_UTL/splitToWords.H000066400000000000000000000100751272335422300167150ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/splitToWords.H * * Modifications by: * * Brian P. Walenz from 2005-JUL-12 to 2014-APR-11 * are Copyright 2005-2006,2012,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2015-AUG-11 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-FEB-25 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef SPLITTOWORDS_H #define SPLITTOWORDS_H class splitToWords { public: splitToWords() { _argWords = 0; _maxWords = 0; _arg = 0L; _maxChars = 0; _cmd = 0L; }; splitToWords(char *cmd) { _argWords = 0; _maxWords = 0; _arg = 0L; _maxChars = 0; _cmd = 0L; split(cmd); }; ~splitToWords() { delete [] _cmd; delete [] _arg; }; void split(char *cmd) { // Step Zero: // // Count the length of the string, in words and in characters. // For simplicity, we overcount words, by just counting white-space. // // Then, allocate space for a temporary copy of the string, and a // set of pointers into the temporary copy (much like argv). // uint32 cmdChars = 1; // 1 == Space for terminating 0 uint32 cmdWords = 2; // 2 == Space for first word and terminating 0L for (char *tmp=cmd; *tmp; tmp++) { cmdWords += (*tmp == ' ') ? 1 : 0; cmdWords += (*tmp == '\t') ? 1 : 0; cmdChars++; } if (cmdChars > _maxChars) { delete [] _cmd; _cmd = new char [cmdChars]; _maxChars = cmdChars; } if (cmdWords > _maxWords) { delete [] _arg; _arg = new char * [cmdWords]; _maxWords = cmdWords; } _argWords = 0; // Step One: // // Determine where the words are in the command string, copying the // string to _cmd and storing words in _arg. // bool isFirst = true; char *cmdI = cmd; char *cmdO = _cmd; while (*cmdI) { // If we are at a non-space character, we are in a word. If // this is the first character in the word, save the word in // the args list. // // Otherwise we are at a space and thus not in a word. Make // all spaces be string terminators, and declare that we are // at the start of a word. // if ((*cmdI != ' ') && (*cmdI != '\t') && (*cmdI != '\n') && (*cmdI != '\r')) { *cmdO = *cmdI; if (isFirst) { _arg[_argWords++] = cmdO; isFirst = false; } } else { *cmdO = 0; isFirst = true; } cmdI++; cmdO++; } // Finish off the list by terminating the last arg, and // terminating the list of args. // *cmdO = 0; _arg[_argWords] = 0L; }; uint32 numWords(void) { return(_argWords); }; char *getWord(uint32 i) { return(_arg[i]); }; char *operator[](uint32 i) { return(_arg[i]); }; int64 operator()(uint32 i) { return(strtoull(_arg[i], NULL, 10)); }; private: uint32 _argWords; uint32 _maxWords; char **_arg; uint32 _maxChars; char *_cmd; }; #endif // SPLITTOWORDS_H canu-1.3/src/AS_UTL/stddev.H000066400000000000000000000272521272335422300155360ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-JUL-23 to 2015-AUG-18 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-27 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-MAR-31 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef STDDEV_H #define STDDEV_H #include "AS_global.H" #include #include using namespace std; // Online mean and std.dev calculation. // B. P. Welford, Technometrics, Vol 4, No 3, Aug 1962 pp 419-420. // http://www2.in.tu-clausthal.de/~zach/teaching/info_literatur/Welford.pdf // Also presented in Knuth Vol 2 (3rd Ed.) pp 232. // template class stdDev { public: stdDev() { _mn = 0.0; _sn = 0.0; _nn = 0; }; ~stdDev() { }; void insert(TT val) { double m0 = _mn; double s0 = _sn; uint32 n0 = _nn + 1; if (_nn & 0x80000000) fprintf(stderr, "ERROR: stdDev has been finalized; can't insert() new value.\n"), exit(1); _mn = m0 + (val - m0) / n0; _sn = s0 + (val - m0) * (val - _mn); _nn = n0; }; void remove(double val) { uint32 n0 = _nn - 1; double m0 = (n0 == 0) ? (0) : ((_nn * _mn - val) / n0); double s0 = _sn - (val - m0) * (val - _mn); if (_nn & 0x80000000) fprintf(stderr, "ERROR: stdDev has been finalized; can't remove() old value.\n"), exit(1); _nn = n0; _mn = m0; _sn = s0; }; void finalize(void) { _sn = stddev(); _nn |= 0x80000000; }; uint32 size(void) { return(_nn & 0x7fffffff); }; double mean(void) { return(_mn); }; double variance(void) { if (_nn & 0x80000000) return(_sn * _sn); else return((_nn < 2) ? (0.0) : (_sn / (_nn-1))); }; double stddev(void) { if (_nn & 0x80000000) return(_sn); else return(sqrt(variance())); }; private: double _mn; // mean double _sn; // "sum of variances" uint32 _nn; // number of items in the set }; // Offline mean and std.dev calculation. Filters outliers. // Does not work well with unsigned types. The 'smallest' compute can underflow. // template void computeStdDev(vector dist, double &mean, double &stddev, bool isSorted=false) { mean = 0; stddev = 0; if (dist.size() == 0) return; // Sort the values. Lets us approximate the stddev for filtering out outliers. if (isSorted == false) sort(dist.begin(), dist.end()); // Approximate the stddev to filter out outliers. This is done by assuming we're normally // distributed, finding the values that would represent 1 standard deviation (about 68.27% of the // data), and using that to find the 5 std.dev. limits. TT median = dist[1 * dist.size() / 2]; TT oneThird = dist[1 * dist.size() / 3]; TT twoThird = dist[2 * dist.size() / 3]; TT approxStd = max(median - oneThird, twoThird - median); TT biggest = median + approxStd * 5; TT smallest = median - approxStd * 5; fprintf(stderr, "computeStdDev %d %d %d %d %d %d\n", median, oneThird, twoThird, approxStd, biggest, smallest); // Now, compute the number of samples within our bounds. And find the mean, too. size_t numSamples = 0; for (size_t x=0; x 1) stddev = sqrt(stddev / (numSamples - 1)); }; // Compute the mode. Once the values are sorted, we just need to scan the list and remember the // most common value. // template void computeMode(vector dist, TT &mode, bool isSorted=false) { mode = 0; if (dist.size() == 0) return; if (isSorted == false) sort(dist.begin(), dist.end()); uint32 modeCnt = 0; TT modeVal = 0; uint32 modeTmpCnt = 0; TT modeTmpVal = 0; for (uint64 x=0; x void computeMedianAbsoluteDeviation(vector dist, TT &median, TT &mad, bool isSorted=false) { median = 0; mad = 0; if (dist.size() == 0) return; if (isSorted == false) sort(dist.begin(), dist.end()); // Technically, if there are an even number of values, the median should be the average of the two // in the middle. median = dist[ dist.size()/2 ]; vector m; for (uint64 ii=0; ii TT computeExponentialMovingAverage(TT alpha, TT ema, TT value) { assert(0.0 <= alpha); assert(alpha <= 1.0); return(alpha * value + (1 - alpha) * ema); }; template class genericStatistics { public: genericStatistics() { _finalized = false; _mean = 0.0; _stddev = 0.0; _mode = 0; _median = 0; _mad = 0; }; ~genericStatistics() { }; void add(TT data) { _finalized = false; _data.push_back(data); }; uint64 numberOfObjects(void) { finalizeData(); return(_data.size()); } double mean(void) { finalizeData(); return(_mean); } double stddev(void) { finalizeData(); return(_stddev); }; TT median(void) { finalizeData(); return(_median); }; TT mad(void) { // Median Absolute Deviation finalizeData(); return(_mad); }; vector &histogram(void) { // Returns pointer to private histogram data finalizeData(); return(&_histogram); }; vector &Nstatistics(void) { // Returns pointer to private N data finalizeData(); return(&_Nstatistics); }; void finalizeData(void) { if (_finalized == true) return; computeStdDev(_data, _mean, _stddev); // Filters out outliers computeMode(_data, _mode); // Mo filtering computeMedianAbsoluteDeviation(_data, _median, _mad); // No filtering _finalized = true; }; private: bool _finalized; vector _data; double _mean; double _stddev; TT _mode; TT _median; TT _mad; vector _histogram; vector _Nstatistics; }; class histogramStatistics { public: histogramStatistics() { _histogramAlloc = 1024 * 1024; _histogramMax = 0; _histogram = new uint64 [_histogramAlloc]; memset(_histogram, 0, sizeof(uint64) * _histogramAlloc); _finalized = false; clearStatistics(); }; ~histogramStatistics() { delete [] _histogram; }; void add(uint64 data, uint32 count=1) { while (_histogramAlloc < data) resizeArray(_histogram, _histogramMax+1, _histogramAlloc, _histogramAlloc * 2, resizeArray_copyData | resizeArray_clearNew); if (_histogramMax < data) _histogramMax = data; _histogram[data] += count; _finalized = false; }; uint64 numberOfObjects(void) { finalizeData(); return(_numObjs); }; double mean(void) { finalizeData(); return(_mean); }; double stddev(void) { finalizeData(); return(_stddev); }; uint64 median(void) { finalizeData(); return(_median); }; uint64 mad(void) { finalizeData(); return(_mad); }; #if 0 vector &histogram(void) { // Returns pointer to private histogram data finalizeData(); return(&_histogram); }; vector &Nstatistics(void) { // Returns pointer to private N data finalizeData(); return(&_Nstatistics); }; #endif void clearStatistics(void) { _numObjs = 0; _mean = 0.0; _stddev = 0.0; _mode = 0; _median = 0; _mad = 0; }; void finalizeData(void) { if (_finalized == true) return; clearStatistics(); // Compute number of objects for (uint64 ii=0; ii <= _histogramMax; ii++) _numObjs += _histogram[ii]; // Compute mean and stddev for (uint64 ii=0; ii <= _histogramMax; ii++) _mean += ii * _histogram[ii]; if (_numObjs > 1) _mean /= _numObjs; for (uint64 ii=0; ii <= _histogramMax; ii++) _stddev += _histogram[ii] * (ii - _mean) * (ii - _mean); if (_numObjs > 1) _stddev = sqrt(_stddev / (_numObjs - 1)); // Compute mode for (uint64 ii=0; ii <= _histogramMax; ii++) if (_histogram[ii] > _histogram[_mode]) _mode = ii; // Compute median and mad for (uint64 ii=0; ii <= _histogramMax; ii++) if (_median < _numObjs / 2) { _median += _histogram[ii]; } else { _median = ii; break; } uint64 *maddata = new uint64 [_histogramAlloc]; memset(maddata, 0, sizeof(uint64) * _histogramAlloc); for (uint64 ii=0; ii <= _histogramMax; ii++) { uint64 mad = (ii < _median) ? (_median - ii) : (ii - _median); maddata[mad] += _histogram[ii]; } for (uint64 ii=0; ii <= _histogramMax; ii++) if (_mad < _numObjs / 2) { _mad += maddata[ii]; } else { _mad = ii; break; } // And, done delete [] maddata; _finalized = true; }; uint64 histogram(uint64 ii) { return(_histogram[ii]); }; uint64 histogramMax(void) { return(_histogramMax); }; void writeHistogram(FILE *F, char *label) { fprintf(F, "#%s\tquantity\n", label); for (uint64 ii=0; ii <= _histogramMax; ii++) fprintf(F, F_U64"\t"F_U64"\n", ii, _histogram[ii]); }; private: bool _finalized; uint64 _histogramAlloc; // Maximum allocated value uint64 _histogramMax; // Maximum valid value uint64 *_histogram; uint64 _numObjs; double _mean; double _stddev; uint64 _mode; uint64 _median; uint64 _mad; }; #endif // STDDEV_H canu-1.3/src/AS_UTL/stddevTest.C000066400000000000000000000057101272335422300163640ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAR-10 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "stddev.H" // g++ -Wall -o stddevTest -I. -I.. stddevTest.C void testInsert(void) { stdDev sdu; stdDev sdi; stdDev sdd; sdu.insert((uint32)2); sdu.insert((uint32)4); sdu.insert((uint32)4); sdu.insert((uint32)4); sdu.insert((uint32)5); sdu.insert((uint32)5); sdu.insert((uint32)7); sdu.insert((uint32)9); sdi.insert((uint32)2); sdi.insert((uint32)4); sdi.insert((uint32)4); sdi.insert((uint32)4); sdi.insert((uint32)5); sdi.insert((uint32)5); sdi.insert((uint32)7); sdi.insert((uint32)9); sdd.insert((uint32)2); sdd.insert((uint32)4); sdd.insert((uint32)4); sdd.insert((uint32)4); sdd.insert((uint32)5); sdd.insert((uint32)5); sdd.insert((uint32)7); sdd.insert((uint32)9); fprintf(stderr, "Expect mean=5, variance=%f, stddev=%f\n", 32.0 / 7.0, sqrt(32.0 / 7.0)); fprintf(stderr, " uint32 size %u mean %f variance %f stddev %f\n", sdu.size(), sdu.mean(), sdu.variance(), sdu.stddev()); fprintf(stderr, " int32 size %u mean %f variance %f stddev %f\n", sdi.size(), sdi.mean(), sdi.variance(), sdi.stddev()); fprintf(stderr, " double size %u mean %f variance %f stddev %f\n", sdd.size(), sdd.mean(), sdd.variance(), sdd.stddev()); assert(sdu.variance() == 32.0 / 7.0); assert(sdi.variance() == 32.0 / 7.0); assert(sdd.variance() == 32.0 / 7.0); fprintf(stderr, "\n\n"); } void testRemove(void) { double values[10] = { 1, 2, 3, 4, 9, 8, 7, 6, 20, 30 }; stdDev sd; fprintf(stderr, "Expect final to be zero, and insert() == remove().\n"); for (int ii=0; ii<10; ii++) { sd.insert(values[ii]); fprintf(stderr, "insert[%2d] mean %8.4f stddev %8.4f\n", ii+1, sd.mean(), sd.stddev()); } assert(sd.mean() == 9.0); fprintf(stderr, "\n"); for (int ii=9; ii>=0; ii--) { sd.remove(values[ii]); fprintf(stderr, "remove[%2d] mean %8.4f stddev %8.4f\n", ii, sd.mean(), sd.stddev()); } assert(sd.mean() == 0.0); assert(sd.stddev() == 0.0); } int main(int argc, char **argv) { testInsert(); testRemove(); exit(0); } canu-1.3/src/AS_UTL/sweatShop.C000066400000000000000000000421521272335422300162110ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/sweatShop.C * * Modifications by: * * Brian P. Walenz from 2006-MAR-02 to 2014-APR-11 * are Copyright 2006,2008,2010-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2015-JUN-24 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "sweatShop.H" #include "timeAndSize.H" #include // pthread scheduling stuff class sweatShopWorker { public: sweatShopWorker() { shop = 0L; threadUserData = 0L; numComputed = 0; workerQueue = 0L; workerQueueLen = 0L; }; sweatShop *shop; void *threadUserData; pthread_t threadID; uint32 numComputed; sweatShopState **workerQueue; uint32 workerQueueLen; }; // This gets created by the loader, passed to the worker, and printed // by the writer. userData is controlled by the user. // class sweatShopState { public: sweatShopState(void *userData) { _user = userData; _computed = false; _next = 0L; }; ~sweatShopState() { }; void *_user; bool _computed; sweatShopState *_next; }; // Simply forwards control to the class void* _sweatshop_loaderThread(void *ss_) { sweatShop *ss = (sweatShop *)ss_; return(ss->loader()); } void* _sweatshop_workerThread(void *sw_) { sweatShopWorker *sw = (sweatShopWorker *)sw_; return(sw->shop->worker(sw)); } void* _sweatshop_writerThread(void *ss_) { sweatShop *ss = (sweatShop *)ss_; return(ss->writer()); } void* _sweatshop_statusThread(void *ss_) { sweatShop *ss = (sweatShop *)ss_; return(ss->status()); } sweatShop::sweatShop(void*(*loaderfcn)(void *G), void (*workerfcn)(void *G, void *T, void *S), void (*writerfcn)(void *G, void *S)) { _userLoader = loaderfcn; _userWorker = workerfcn; _userWriter = writerfcn; _globalUserData = 0L; _writerP = 0L; _workerP = 0L; _loaderP = 0L; _showStatus = false; _loaderQueueSize = 1024; _loaderQueueMax = 10240; _loaderQueueMin = 4; // _numberOfWorkers * 2, reset when that changes _loaderBatchSize = 1; _workerBatchSize = 1; _writerQueueSize = 4096; _writerQueueMax = 10240; _numberOfWorkers = 2; _workerData = 0L; _numberLoaded = 0; _numberComputed = 0; _numberOutput = 0; } sweatShop::~sweatShop() { delete [] _workerData; } void sweatShop::setThreadData(uint32 t, void *x) { if (_workerData == 0L) _workerData = new sweatShopWorker [_numberOfWorkers]; if (t >= _numberOfWorkers) fprintf(stderr, "sweatShop::setThreadData()-- worker ID "F_U32" more than number of workers="F_U32"\n", t, _numberOfWorkers), exit(1); _workerData[t].threadUserData = x; } // Build a list of states to add in one swoop // void sweatShop::loaderSave(sweatShopState *&tail, sweatShopState *&head, sweatShopState *thisState) { thisState->_next = 0L; if (tail) { head->_next = thisState; head = thisState; } else { tail = head = thisState; } _numberLoaded++; } // Add a bunch of new states to the queue. // void sweatShop::loaderAppend(sweatShopState *&tail, sweatShopState *&head) { int err; if ((tail == 0L) || (head == 0L)) return; err = pthread_mutex_lock(&_stateMutex); if (err != 0) fprintf(stderr, "sweatShop::loaderAppend()-- Failed to lock mutex (%d). Fail.\n", err), exit(1); if (_loaderP == 0L) { _writerP = tail; _workerP = tail; _loaderP = head; } else { _loaderP->_next = tail; } _loaderP = head; err = pthread_mutex_unlock(&_stateMutex); if (err != 0) fprintf(stderr, "sweatShop::loaderAppend()-- Failed to unlock mutex (%d). Fail.\n", err), exit(1); tail = 0L; head = 0L; } void* sweatShop::loader(void) { struct timespec naptime; naptime.tv_sec = 0; naptime.tv_nsec = 166666666ULL; // 1/6 second // We can batch several loads together before we push them onto the // queue, this should reduce the number of times the loader needs to // lock the queue. // // But it also increases the latency, so it's disabled by default. // sweatShopState *tail = 0L; // The first thing loaded sweatShopState *head = 0L; // The last thing loaded uint32 numLoaded = 0; bool moreToLoad = true; while (moreToLoad) { // Zzzzzzz.... while (_numberLoaded > _numberComputed + _loaderQueueSize) nanosleep(&naptime, 0L); sweatShopState *thisState = new sweatShopState((*_userLoader)(_globalUserData)); // If we actually loaded a new state, add it // if (thisState->_user) { loaderSave(tail, head, thisState); numLoaded++; if (numLoaded >= _loaderBatchSize) loaderAppend(tail, head); } else { // Didn't read, must be all done! Push on the end-of-input marker state. // loaderSave(tail, head, new sweatShopState(0L)); loaderAppend(tail, head); moreToLoad = false; delete thisState; } } //fprintf(stderr, "sweatShop::reader exits.\n"); return(0L); } void* sweatShop::worker(sweatShopWorker *workerData) { struct timespec naptime; naptime.tv_sec = 0; naptime.tv_nsec = 50000000ULL; bool moreToCompute = true; int err; while (moreToCompute) { // Usually beacuse some worker is taking a long time, and the // output queue isn't big enough. // while (_numberOutput + _writerQueueSize < _numberComputed) nanosleep(&naptime, 0L); // Grab the next state. We don't grab it if it's the last in the // queue (else we would fall off the end) UNLESS it really is the // last one. // err = pthread_mutex_lock(&_stateMutex); if (err != 0) fprintf(stderr, "sweatShop::worker()-- Failed to lock mutex (%d). Fail.\n", err), exit(1); for (workerData->workerQueueLen = 0; ((workerData->workerQueueLen < _workerBatchSize) && (_workerP) && ((_workerP->_next != 0L) || (_workerP->_user == 0L))); workerData->workerQueueLen++) { workerData->workerQueue[workerData->workerQueueLen] = _workerP; _workerP = _workerP->_next; } if (_workerP == 0L) moreToCompute = false; err = pthread_mutex_unlock(&_stateMutex); if (err != 0) fprintf(stderr, "sweatShop::worler()-- Failed to lock mutex (%d). Fail.\n", err), exit(1); if (workerData->workerQueueLen == 0) { // No work, sleep a bit to prevent thrashing the mutex and resume. nanosleep(&naptime, 0L); continue; } // Execute // for (uint32 x=0; xworkerQueueLen; x++) { sweatShopState *ts = workerData->workerQueue[x]; if (ts && ts->_user) { (*_userWorker)(_globalUserData, workerData->threadUserData, ts->_user); ts->_computed = true; workerData->numComputed++; } else { // When we really do run out of stuff to do, we'll end up here // (only one thread will end up in the other case, with // something to do and moreToCompute=false). If it's actually // the end, skip the sleep and just get outta here. // if (moreToCompute == true) { fprintf(stderr, "WARNING! Worker is sleeping because the reader is slow!\n"); nanosleep(&naptime, 0L); } } } } //fprintf(stderr, "sweatShop::worker exits.\n"); return(0L); } void* sweatShop::writer(void) { sweatShopState *deleteState = 0L; // Wait for output to appear, then write. // while (_writerP && _writerP->_user) { if (_writerP->_computed == false) { // Wait for a slow computation. struct timespec naptime; naptime.tv_sec = 0; naptime.tv_nsec = 5000000ULL; //fprintf(stderr, "Writer waits for slow thread at "F_U64".\n", _numberOutput); nanosleep(&naptime, 0L); } else if (_writerP->_next == 0L) { // Wait for the input. struct timespec naptime; naptime.tv_sec = 0; naptime.tv_nsec = 5000000ULL; //fprintf(stderr, "Writer waits for all threads at "F_U64".\n", _numberOutput); nanosleep(&naptime, 0L); } else { (*_userWriter)(_globalUserData, _writerP->_user); _numberOutput++; deleteState = _writerP; _writerP = _writerP->_next; delete deleteState; } } // Tell status to stop. _writerP = 0L; //fprintf(stderr, "sweatShop::writer exits.\n"); return(0L); } // This thread not only shows a status message, but it also updates the critical shared variable // _numberComputed. Worker threads use this to throttle themselves. Thus, even if _showStatus is // not set, and this thread doesn't _appear_ to be doing anything useful....it is. // void* sweatShop::status(void) { struct timespec naptime; naptime.tv_sec = 0; naptime.tv_nsec = 250000000ULL; double startTime = getTime() - 0.001; double thisTime = 0; uint64 deltaOut = 0; uint64 deltaCPU = 0; double cpuPerSec = 0; uint64 readjustAt = 16384; while (_writerP) { uint32 nc = 0; for (uint32 i=0; i<_numberOfWorkers; i++) nc += _workerData[i].numComputed; _numberComputed = nc; deltaOut = deltaCPU = 0; thisTime = getTime(); if (_numberComputed > _numberOutput) deltaOut = _numberComputed - _numberOutput; if (_numberLoaded > _numberComputed) deltaCPU = _numberLoaded - _numberComputed; cpuPerSec = _numberComputed / (thisTime - startTime); if (_showStatus) { fprintf(stderr, " %6.1f/s - %8"F_U64P" loaded; %8"F_U64P" queued for compute; %08"F_U64P" finished; %8"F_U64P" written; %8"F_U64P" queued for output)\r", cpuPerSec, _numberLoaded, deltaCPU, _numberComputed, _numberOutput, deltaOut); fflush(stderr); } // Readjust queue sizes based on current performance, but don't let it get too big or small. // In particular, don't let it get below 2*numberOfWorkers. // if (_numberComputed > readjustAt) { readjustAt += (uint64)(2 * cpuPerSec); _loaderQueueSize = (uint32)(5 * cpuPerSec); } if (_loaderQueueSize < _loaderQueueMin) _loaderQueueSize = _loaderQueueMin; if (_loaderQueueSize < 2 * _numberOfWorkers) _loaderQueueSize = 2 * _numberOfWorkers; if (_loaderQueueSize > _loaderQueueMax) _loaderQueueSize = _loaderQueueMax; nanosleep(&naptime, 0L); } if (_showStatus) { thisTime = getTime(); if (_numberComputed > _numberOutput) deltaOut = _numberComputed - _numberOutput; if (_numberLoaded > _numberComputed) deltaCPU = _numberLoaded - _numberComputed; cpuPerSec = _numberComputed / (thisTime - startTime); fprintf(stderr, " %6.1f/s - %08"F_U64P" queued for compute; %08"F_U64P" finished; %08"F_U64P" queued for output)\n", cpuPerSec, deltaCPU, _numberComputed, deltaOut); } //fprintf(stderr, "sweatShop::status exits.\n"); return(0L); } void sweatShop::run(void *user, bool beVerbose) { pthread_attr_t threadAttr; pthread_t threadIDloader; pthread_t threadIDwriter; pthread_t threadIDstats; #if 0 int threadSchedPolicy = 0; struct sched_param threadSchedParamDef; struct sched_param threadSchedParamMax; #endif int err = 0; _globalUserData = user; _showStatus = beVerbose; // Configure everything ahead of time. if (_workerBatchSize < 1) _workerBatchSize = 1; if (_workerData == 0L) _workerData = new sweatShopWorker [_numberOfWorkers]; for (uint32 i=0; i<_numberOfWorkers; i++) { _workerData[i].shop = this; _workerData[i].workerQueue = new sweatShopState * [_workerBatchSize]; } // Open the doors. errno = 0; err = pthread_mutex_init(&_stateMutex, NULL); if (err) fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (state mutex): %s.\n", strerror(err)), exit(1); err = pthread_attr_init(&threadAttr); if (err) fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (attr init): %s.\n", strerror(err)), exit(1); err = pthread_attr_setscope(&threadAttr, PTHREAD_SCOPE_SYSTEM); if (err) fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (set scope): %s.\n", strerror(err)), exit(1); err = pthread_attr_setdetachstate(&threadAttr, PTHREAD_CREATE_JOINABLE); if (err) fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (joinable): %s.\n", strerror(err)), exit(1); #if 0 err = pthread_attr_getschedparam(&threadAttr, &threadSchedParamDef); if (err) fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (get default param): %s.\n", strerror(err)), exit(1); err = pthread_attr_getschedparam(&threadAttr, &threadSchedParamMax); if (err) fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (get max param): %s.\n", strerror(err)), exit(1); #endif // SCHED_RR needs root privs to run on FreeBSD. // //err = pthread_attr_setschedpolicy(&threadAttr, SCHED_RR); //if (err) // fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (sched policy): %s.\n", strerror(err)), exit(1); #if 0 err = pthread_attr_getschedpolicy(&threadAttr, &threadSchedPolicy); if (err) fprintf(stderr, "sweatShop::run()-- Failed to configure pthreads (sched policy): %s.\n", strerror(err)), exit(1); errno = 0; threadSchedParamMax.sched_priority = sched_get_priority_max(threadSchedPolicy); if (errno) fprintf(stderr, "sweatShop::run()-- WARNING: Failed to configure pthreads (set max param priority): %s.\n", strerror(errno)); // Fire off the loader err = pthread_attr_setschedparam(&threadAttr, &threadSchedParamMax); if (err) fprintf(stderr, "sweatShop::run()-- Failed to set loader priority: %s.\n", strerror(err)), exit(1); #endif err = pthread_create(&threadIDloader, &threadAttr, _sweatshop_loaderThread, this); if (err) fprintf(stderr, "sweatShop::run()-- Failed to launch loader thread: %s.\n", strerror(err)), exit(1); // Wait for it to actually load something (otherwise all the // workers immediately go home) while (!_writerP && !_workerP && !_loaderP) { struct timespec naptime; naptime.tv_sec = 0; naptime.tv_nsec = 250000ULL; nanosleep(&naptime, 0L); } // Start the statistics and writer #if 0 err = pthread_attr_setschedparam(&threadAttr, &threadSchedParamMax); if (err) fprintf(stderr, "sweatShop::run()-- Failed to set status and writer priority: %s.\n", strerror(err)), exit(1); #endif err = pthread_create(&threadIDstats, &threadAttr, _sweatshop_statusThread, this); if (err) fprintf(stderr, "sweatShop::run()-- Failed to launch status thread: %s.\n", strerror(err)), exit(1); err = pthread_create(&threadIDwriter, &threadAttr, _sweatshop_writerThread, this); if (err) fprintf(stderr, "sweatShop::run()-- Failed to launch writer thread: %s.\n", strerror(err)), exit(1); // And some labor #if 0 err = pthread_attr_setschedparam(&threadAttr, &threadSchedParamDef); if (err) fprintf(stderr, "sweatShop::run()-- Failed to set worker priority: %s.\n", strerror(err)), exit(1); #endif for (uint32 i=0; i<_numberOfWorkers; i++) { err = pthread_create(&_workerData[i].threadID, &threadAttr, _sweatshop_workerThread, _workerData + i); if (err) fprintf(stderr, "sweatShop::run()-- Failed to launch worker thread "F_U32": %s.\n", i, strerror(err)), exit(1); } // Now sit back and relax. err = pthread_join(threadIDloader, 0L); if (err) fprintf(stderr, "sweatShop::run()-- Failed to join loader thread: %s.\n", strerror(err)), exit(1); err = pthread_join(threadIDwriter, 0L); if (err) fprintf(stderr, "sweatShop::run()-- Failed to join writer thread: %s.\n", strerror(err)), exit(1); err = pthread_join(threadIDstats, 0L); if (err) fprintf(stderr, "sweatShop::run()-- Failed to join status thread: %s.\n", strerror(err)), exit(1); for (uint32 i=0; i<_numberOfWorkers; i++) { err = pthread_join(_workerData[i].threadID, 0L); if (err) fprintf(stderr, "sweatShop::run()-- Failed to join worker thread "F_U32": %s.\n", i, strerror(err)), exit(1); } // Cleanup. delete _loaderP; _loaderP = _workerP = _writerP = 0L; } canu-1.3/src/AS_UTL/sweatShop.H000066400000000000000000000074131272335422300162170ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libutil/sweatShop.H * * Modifications by: * * Brian P. Walenz from 2006-MAR-02 to 2014-APR-11 * are Copyright 2006,2008,2010-2011,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef SWEATSHOP_H #define SWEATSHOP_H #include #include #include "AS_global.H" class sweatShopWorker; class sweatShopState; class sweatShop { public: sweatShop(void*(*loaderfcn)(void *G), void (*workerfcn)(void *G, void *T, void *S), void (*writerfcn)(void *G, void *S)); ~sweatShop(); void setNumberOfWorkers(uint32 x) { _numberOfWorkers = x; _loaderQueueMin = x * 2; }; void setThreadData(uint32 t, void *x); void setLoaderBatchSize(uint32 batchSize) { _loaderBatchSize = batchSize; }; void setLoaderQueueSize(uint32 queueSize) { _loaderQueueSize = queueSize; _loaderQueueMax = queueSize; }; void setWorkerBatchSize(uint32 batchSize) { _workerBatchSize = batchSize; }; void setWriterQueueSize(uint32 queueSize) { _writerQueueSize = queueSize; _writerQueueMax = queueSize; }; void run(void *user=0L, bool beVerbose=false); private: // Stubs that forward control from the c-based pthread to this class friend void *_sweatshop_loaderThread(void *ss); friend void *_sweatshop_workerThread(void *ss); friend void *_sweatshop_writerThread(void *ss); friend void *_sweatshop_statusThread(void *ss); // The threaded routines void *loader(void); void *worker(sweatShopWorker *workerData); void *writer(void); void *status(void); // Utilities for the loader thread //void loaderAdd(sweatShopState *thisState); void loaderSave(sweatShopState *&tail, sweatShopState *&head, sweatShopState *thisState); void loaderAppend(sweatShopState *&tail, sweatShopState *&head); pthread_mutex_t _stateMutex; void *(*_userLoader)(void *global); void (*_userWorker)(void *global, void *thread, void *thing); void (*_userWriter)(void *global, void *thing); void *_globalUserData; sweatShopState *_writerP; // Where output takes stuff from, the tail sweatShopState *_workerP; // Where computes happen, the middle sweatShopState *_loaderP; // Where input is put, the head bool _showStatus; uint32 _loaderQueueSize, _loaderQueueMin, _loaderQueueMax; uint32 _loaderBatchSize; uint32 _workerBatchSize; uint32 _writerQueueSize, _writerQueueMax; uint32 _numberOfWorkers; sweatShopWorker *_workerData; uint64 _numberLoaded; uint64 _numberComputed; uint64 _numberOutput; }; #endif // SWEATSHOP_H canu-1.3/src/AS_UTL/testRand.C000066400000000000000000000043471272335422300160240ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2008-JUN-27 to 2013-AUG-01 * are Copyright 2008,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz beginning on 2015-OCT-12 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include "AS_UTL_rand.H" #define ITERATIONS 1000000 int main(void){ double sumx = 0.0, sumx2 = 0.0, sumx3 = 0.0; int i; int Seed = getpid(); double num; srand48(Seed); /* Initialize Random Number Generator */ for(i = 0; i < ITERATIONS; i++){ double rand = GaussRandomNormalized_AS(); double mult = rand; sumx += mult; mult *= rand; sumx2 += mult; mult *= rand; sumx3 += mult; } num = (double)ITERATIONS; fprintf(stderr,"***** Normalized Gaussian *****"); fprintf(stderr,"* avg = %f avg2 = %f avg3 = %f\n", sumx/num, sumx2/num, sumx3/num); sumx = 0.0; sumx2 = 0.0; sumx3 = 0.0; for(i = 0; i < ITERATIONS; i++){ double rand = GaussRandom_AS(0.0, 5.0); double mult = rand; sumx += mult; mult *= rand; sumx2 += mult; mult *= rand; sumx3 += mult; } num = (double)ITERATIONS; sumx = sumx/num; sumx2 = sqrt(sumx2/num); sumx3 = pow(sumx3/num, 0.33); fprintf(stderr,"***** Gaussian with STDEV 5.0 *****"); fprintf(stderr,"* avg = %f sqrt avg2 = %f curt avg3 = %f\n", sumx, sumx2, sumx3); return 0; } canu-1.3/src/AS_UTL/timeAndSize.C000066400000000000000000000034041272335422300164450ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include #include #include double getTime(void) { struct timeval tp; gettimeofday(&tp, NULL); return(tp.tv_sec + (double)tp.tv_usec / 1000000.0); } uint64 getProcessSizeCurrent(void) { struct rusage ru; uint64 sz = 0; errno = 0; if (getrusage(RUSAGE_SELF, &ru) == -1) { fprintf(stderr, "getProcessSizeCurrent()-- getrusage(RUSAGE_SELF, ...) failed: %s\n", strerror(errno)); } else { sz = ru.ru_maxrss; sz *= 1024; } return(sz); } uint64 getProcessSizeLimit(void) { struct rlimit rlp; uint64 sz = ~uint64ZERO; errno = 0; if (getrlimit(RLIMIT_DATA, &rlp) == -1) { fprintf(stderr, "getProcessSizeLimit()-- getrlimit(RLIMIT_DATA, ...) failed: %s\n", strerror(errno)); } else { sz = rlp.rlim_cur; } return(sz); } canu-1.3/src/AS_UTL/timeAndSize.H000066400000000000000000000020101272335422300164420ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" double getTime(void); uint64 getProcessSizeCurrent(void); uint64 getProcessSizeLimit(void); canu-1.3/src/AS_global.C000066400000000000000000000120431272335422300150240ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2007-AUG-03 to 2013-AUG-01 * are Copyright 2007-2009,2011-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2009-MAR-06 * are Copyright 2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2015-MAR-03 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-08 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "canu_version.H" #include "AS_UTL_stackTrace.H" #ifdef X86_GCC_LINUX #include #endif #ifdef _GLIBCXX_PARALLEL #include #include #endif // We take argc and argv, so, maybe, eventually, we'll want to parse // something out of there. We return argc in case what we parse we // want to remove. // int AS_configure(int argc, char **argv) { char *p = NULL; int i, j; #ifdef X86_GCC_LINUX // Set the x86 FPU control word to force double precision rounding // rather than `extended' precision rounding. This causes base // calls and quality values on x86 GCC-Linux (tested on RedHat // Linux) machines to be identical to those on IEEE conforming UNIX // machines. // fpu_control_t fpu_cw = ( _FPU_DEFAULT & ~_FPU_EXTENDED ) | _FPU_DOUBLE; _FPU_SETCW( fpu_cw ); #endif #ifdef _GLIBCXX_PARALLEL_SETTINGS_H __gnu_parallel::_Settings s = __gnu_parallel::_Settings::get(); // Force all algorithms to be parallel. // Force some algs to be sequential by using a tag, eg: // sort(a, a+end, __gnu_parallel::sequential_tag()); // //s.algorithm_strategy = __gnu_parallel::force_parallel; // The default seems to be 1000, way too small for us. s.sort_minimal_n = 128 * 1024; // The default is MWMS, which, at least on FreeBSD 8.2 w/gcc46, is NOT inplace. // Then again, the others also appear to be NOT inplace as well. //s.sort_algorithm = __gnu_parallel::MWMS; //s.sort_algorithm = __gnu_parallel::QS_BALANCED; //s.sort_algorithm = __gnu_parallel::QS; __gnu_parallel::_Settings::set(s); #endif // Install a signal handler to catch seg faults and errors. AS_UTL_installCrashCatcher(); // // Et cetera. // for (i=0; i 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 2))) // Not by default! //#ifndef _GLIBCXX_PARALLEL //#define _GLIBCXX_PARALLEL //#endif #else #undef _GLIBCXX_PARALLEL #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef TRUE #define TRUE true #endif #ifndef FALSE #define FALSE false #endif #ifndef MIN #define MIN(a,b) (((a) < (b)) ? (a) : (b)) #endif #ifndef MAX #define MAX(a,b) (((a) > (b)) ? (a) : (b)) #endif typedef int8_t int8; typedef int16_t int16; typedef int32_t int32; typedef int64_t int64; typedef uint8_t uint8; typedef uint16_t uint16; typedef uint32_t uint32; typedef uint64_t uint64; //typedef void* PtrT; // Correct? Or should it be LLU and LU? #define uint64NUMBER(X) X ## LU #define uint32NUMBER(X) X ## U #define uint64ZERO uint64NUMBER(0x0000000000000000) #define uint64ONE uint64NUMBER(0x0000000000000001) #define uint64MAX uint64NUMBER(0xffffffffffffffff) #define uint64MASK(X) ((~uint64ZERO) >> (64 - (X))) #define uint32ZERO uint32NUMBER(0x00000000) #define uint32ONE uint32NUMBER(0x00000001) #define uint32MAX uint32NUMBER(0xffffffff) #define uint32MASK(X) ((~uint32ZERO) >> (32 - (X))) #define uint16ZERO (0x0000) #define uint16ONE (0x0001) #define uint16MAX (0xffff) #define uint16MASK(X) ((~uint16ZERO) >> (16 - (X))) #define uint8ZERO (0x00) #define uint8ONE (0x01) #define uint8MAX (0xff) #define uint8MASK(X) ((~uint8ZERO) >> (8 - (X))) #define strtouint32(N) (uint32)strtoul (N, NULL, 10) #define strtouint64(N) (uint64)strtoull(N, NULL, 10) #define strtodouble(N) (double)strtod (N, NULL) //inline uint32 strtouint32(char *N, O) (uint32)strtoul (N, O, 10) //inline uint64 strtouint64(char *N, O) (uint64)strtoull(N, O, 10) //inline double strtodouble(char *N, O) (double)strtod (N, O) // Pointers #define F_PTR "0x%016p" // Characters #define F_C "%c" #define F_CP "c" #define F_CI "%*c" // Strings #define F_STR "%s" #define F_STRP "s" #define F_STRI "%*s" // Integers #define F_S16 "%"PRId16 #define F_S16P PRId16 #define F_S16I "%*"PRId16 #define F_U16 "%"PRIu16 #define F_U16P PRIu16 #define F_U16I "%*"PRIu16 #define F_S32 "%"PRId32 #define F_S32P PRId32 #define F_S32I "%*"PRId32 #define F_U32 "%"PRIu32 #define F_U32P PRIu32 #define F_U32I "%*"PRIu32 #define F_S64 "%"PRId64 #define F_S64P PRId64 #define F_S64I "%*"PRId64 #define F_U64 "%"PRIu64 #define F_U64P PRIu64 #define F_U64I "%*"PRIu64 #define F_X64 "%016"PRIx64 #define F_X64P PRIx64 #define F_X64I "%*"PRIx64 // Floating points #define F_F32 "%f" #define F_F32P "f" #define F_F32I "%*f" #define F_F64 "%lf" #define F_F64P "lf" #define F_F64I "%*lf" // Standard typedefs #define F_SIZE_T "%zu" #define F_SIZE_TP "zu" #define F_SIZE_TI "%*zu" #define F_OFF_T F_S64 #define F_OFF_TP F_S64P #define F_OFF_TI F_S64I typedef uintptr_t INTPTR; // These are used to pad various structs to specific sizes #if ULONG_MAX == 0xffffffff #define TRUE32BIT #else #define TRUE64BIT #endif #if defined(_FILE_OFFSET_BITS) && (_FILE_OFFSET_BITS == 32) #error I do not support 32-bit off_t. #endif // Enable troublesome asserts. These typically have work arounds, and occasionally trigger. // They're really only useful if the assembly can be debugged. #undef AGGRESSIVE_ASSERT // perl's chomp is pretty nice // Not a great place to put this, but it's getting used all over. #ifndef chomp #define chomp(S) { char *t=(S); while (*t) t++; t--; while (t >= S && isspace(*t)) *t--=0; } #endif #ifndef munch #define munch(S) { while (*(S) && isspace(*(S))) (S)++; } #endif #ifndef crunch #define crunch(S) { while (*(S) && !isspace(*(S))) (S)++; } #endif int AS_configure(int argc, char **argv); // These macros are use to eliminate inter-platform differnces between // calculated results #define DBL_TO_INT(X) ((int)((1.0+16.0*DBL_EPSILON)*(X))) #define ROUNDPOS(X) (DBL_TO_INT((X)+0.5) ) #define ROUND(X) (((X)>0.0) ? ROUNDPOS(X) : -ROUNDPOS(-(X)) ) #define ZERO_PLUS ( 16.0*DBL_EPSILON) #define ZERO_MINUS (-16.0*DBL_EPSILON) #define ONE_PLUS (1.0+ZERO_PLUS) #define ONE_MINUS (1.0+ZERO_MINUS) #define INT_EQ_DBL(I,D) (fabs((double)(I)-(D)) < 16.0*DBL_EPSILON ) #define DBL_EQ_DBL(A,B) (fabs((A)-(B))<16.0*DBL_EPSILON) // Tell gcc (and others, maybe) about unused parameters. This is important for gcc (especially // newer ones) that complain about unused parameters. Thanks to ideasman42 at // http://stackoverflow.com/questions/3599160/unused-parameter-warnings-in-c-code. #ifdef __GNUC__ #define UNUSED(x) UNUSED_ ## x __attribute__((__unused__)) #else #define UNUSED(x) UNUSED_ ## x #endif #include "AS_UTL_alloc.H" // Stubs for ignoring OpenMP on clang. #ifdef BROKEN_CLANG_OpenMP static void omp_set_dynamic(int x) { #pragma unused(x) } static int omp_get_max_threads(void) { return(1); } static void omp_set_num_threads(int x) { #pragma unused(x) } static int omp_get_num_threads(void) { return(1); } static int omp_get_thread_num(void) { return(0); } typedef int omp_lock_t; static void omp_init_lock(omp_lock_t *a) { #pragma unused(a) } static void omp_set_lock(omp_lock_t *a) { #pragma unused(a) } static void omp_unset_lock(omp_lock_t *a) { #pragma unused(a) } static void omp_destroy_lock(omp_lock_t *a) { #pragma unused(a) } #endif #endif canu-1.3/src/Makefile000066400000000000000000000610241272335422300145400ustar00rootroot00000000000000# boilermake: A reusable, but flexible, boilerplate Makefile. # # Copyright 2008, 2009, 2010 Dan Moulding, Alan T. DeKok # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # Caution: Don't edit this Makefile! Create your own main.mk and other # submakefiles, which will be included by this Makefile. # Only edit this if you need to modify boilermake's behavior (fix # bugs, add features, etc). # Note: Parameterized "functions" in this makefile that are marked with # "USE WITH EVAL" are only useful in conjuction with eval. This is # because those functions result in a block of Makefile syntax that must # be evaluated after expansion. Since they must be used with eval, most # instances of "$" within them need to be escaped with a second "$" to # accomodate the double expansion that occurs when eval is invoked. # ADD_CLEAN_RULE - Parameterized "function" that adds a new rule and phony # target for cleaning the specified target (removing its build-generated # files). # # USE WITH EVAL # define ADD_CLEAN_RULE clean: clean_${1} .PHONY: clean_${1} clean_${1}: $$(strip rm -f ${TARGET_DIR}/${1} $${${1}_OBJS:%.o=%.[doP]}) $${${1}_POSTCLEAN} endef # ADD_OBJECT_RULE - Parameterized "function" that adds a pattern rule for # building object files from source files with the filename extension # specified in the second argument. The first argument must be the name of the # base directory where the object files should reside (such that the portion # of the path after the base directory will match the path to corresponding # source files). The third argument must contain the rules used to compile the # source files into object code form. # # USE WITH EVAL # define ADD_OBJECT_RULE ${1}/%.o: ${2} ${3} endef # ADD_TARGET_RULE - Parameterized "function" that adds a new target to the # Makefile. The target may be an executable or a library. The two allowable # types of targets are distinguished based on the name: library targets must # end with the traditional ".a" extension. # # USE WITH EVAL # define ADD_TARGET_RULE ifeq "$$(suffix ${1})" ".a" # Add a target for creating a static library. $${TARGET_DIR}/${1}: $${${1}_OBJS} @mkdir -p $$(dir $$@) $$(strip $${AR} $${ARFLAGS} $$@ $${${1}_OBJS}) $${${1}_POSTMAKE} else ifeq "$$(suffix ${1})" ".jar" # Add a target for copying files. Actually, just untars the files, kind # of ugly, but the only way I could discover to get the lib/ files installed. $${TARGET_DIR}/${1}: $${${1}} @mkdir -p $$(dir $$@) $$(strip tar -C $$(dir $$@) -xf $${${1}_SOURCES}) else # Add a target for linking an executable. First, attempt to select the # appropriate front-end to use for linking. This might not choose the # right one (e.g. if linking with a C++ static library, but all other # sources are C sources), so the user makefile is allowed to specify a # linker to be used for each target. ifeq "$$(strip $${${1}_LINKER})" "" # No linker was explicitly specified to be used for this target. If # there are any C++ sources for this target, use the C++ compiler. # For all other targets, default to using the C compiler. ifneq "$$(strip $$(filter $${CXX_SRC_EXTS},$${${1}_SOURCES}))" "" ${1}_LINKER = $${CXX} else ${1}_LINKER = $${CC} endif endif $${TARGET_DIR}/${1}: $${${1}_OBJS} $${${1}_PREREQS} @mkdir -p $$(dir $$@) $$(strip $${${1}_LINKER} -o $$@ $${LDFLAGS} $${${1}_LDFLAGS} \ $${${1}_OBJS} $${LDLIBS} $${${1}_LDLIBS}) $${${1}_POSTMAKE} endif endif endef # CANONICAL_PATH - Given one or more paths, converts the paths to the canonical # form. The canonical form is the path, relative to the project's top-level # directory (the directory from which "make" is run), and without # any "./" or "../" sequences. For paths that are not located below the # top-level directory, the canonical form is the absolute path (i.e. from # the root of the filesystem) also without "./" or "../" sequences. define CANONICAL_PATH $(patsubst ${CURDIR}/%,%,$(abspath ${1})) endef # COMPILE_C_CMDS - Commands for compiling C source code. define COMPILE_C_CMDS @mkdir -p $(dir $@) $(strip ${CC} -o $@ -c -MD ${CFLAGS} ${SRC_CFLAGS} ${INCDIRS} \ ${SRC_INCDIRS} ${SRC_DEFS} ${DEFS} $<) @cp ${@:%$(suffix $@)=%.d} ${@:%$(suffix $@)=%.P}; \ sed -e 's/#.*//' -e 's/^[^:]*: *//' -e 's/ *\\$$//' \ -e '/^$$/ d' -e 's/$$/ :/' < ${@:%$(suffix $@)=%.d} \ >> ${@:%$(suffix $@)=%.P}; \ rm -f ${@:%$(suffix $@)=%.d} endef # COMPILE_CXX_CMDS - Commands for compiling C++ source code. define COMPILE_CXX_CMDS @mkdir -p $(dir $@) $(strip ${CXX} -o $@ -c -MD ${CXXFLAGS} ${SRC_CXXFLAGS} ${INCDIRS} \ ${SRC_INCDIRS} ${SRC_DEFS} ${DEFS} $<) @cp ${@:%$(suffix $@)=%.d} ${@:%$(suffix $@)=%.P}; \ sed -e 's/#.*//' -e 's/^[^:]*: *//' -e 's/ *\\$$//' \ -e '/^$$/ d' -e 's/$$/ :/' < ${@:%$(suffix $@)=%.d} \ >> ${@:%$(suffix $@)=%.P}; \ rm -f ${@:%$(suffix $@)=%.d} endef # INCLUDE_SUBMAKEFILE - Parameterized "function" that includes a new # "submakefile" fragment into the overall Makefile. It also recursively # includes all submakefiles of the specified submakefile fragment. # # USE WITH EVAL # define INCLUDE_SUBMAKEFILE # Initialize all variables that can be defined by a makefile fragment, then # include the specified makefile fragment. TARGET := TGT_CFLAGS := TGT_CXXFLAGS := TGT_DEFS := TGT_INCDIRS := TGT_LDFLAGS := TGT_LDLIBS := TGT_LINKER := TGT_POSTCLEAN := TGT_POSTMAKE := TGT_PREREQS := SOURCES := SRC_CFLAGS := SRC_CXXFLAGS := SRC_DEFS := SRC_INCDIRS := SUBMAKEFILES := # A directory stack is maintained so that the correct paths are used as we # recursively include all submakefiles. Get the makefile's directory and # push it onto the stack. DIR := $(call CANONICAL_PATH,$(dir ${1})) DIR_STACK := $$(call PUSH,$${DIR_STACK},$${DIR}) include ${1} # Initialize internal local variables. OBJS := # Determine which target this makefile's variables apply to. A stack is # used to keep track of which target is the "current" target as we # recursively include other submakefiles. ifneq "$$(strip $${TARGET})" "" # This makefile defined a new target. Target variables defined by this # makefile apply to this new target. Initialize the target's variables. TGT := $$(strip $${TARGET}) ALL_TGTS += $${TGT} $${TGT}_CFLAGS := $${TGT_CFLAGS} $${TGT}_CXXFLAGS := $${TGT_CXXFLAGS} $${TGT}_DEFS := $${TGT_DEFS} $${TGT}_DEPS := TGT_INCDIRS := $$(call QUALIFY_PATH,$${DIR},$${TGT_INCDIRS}) TGT_INCDIRS := $$(call CANONICAL_PATH,$${TGT_INCDIRS}) $${TGT}_INCDIRS := $${TGT_INCDIRS} $${TGT}_LDFLAGS := $${TGT_LDFLAGS} $${TGT}_LDLIBS := $${TGT_LDLIBS} $${TGT}_LINKER := $${TGT_LINKER} $${TGT}_OBJS := $${TGT}_POSTCLEAN := $${TGT_POSTCLEAN} $${TGT}_POSTMAKE := $${TGT_POSTMAKE} $${TGT}_PREREQS := $$(addprefix $${TARGET_DIR}/,$${TGT_PREREQS}) $${TGT}_SOURCES := else # The values defined by this makefile apply to the the "current" target # as determined by which target is at the top of the stack. TGT := $$(strip $$(call PEEK,$${TGT_STACK})) $${TGT}_CFLAGS += $${TGT_CFLAGS} $${TGT}_CXXFLAGS += $${TGT_CXXFLAGS} $${TGT}_DEFS += $${TGT_DEFS} TGT_INCDIRS := $$(call QUALIFY_PATH,$${DIR},$${TGT_INCDIRS}) TGT_INCDIRS := $$(call CANONICAL_PATH,$${TGT_INCDIRS}) $${TGT}_INCDIRS += $${TGT_INCDIRS} $${TGT}_LDFLAGS += $${TGT_LDFLAGS} $${TGT}_LDLIBS += $${TGT_LDLIBS} $${TGT}_POSTCLEAN += $${TGT_POSTCLEAN} $${TGT}_POSTMAKE += $${TGT_POSTMAKE} $${TGT}_PREREQS += $${TGT_PREREQS} endif # Push the current target onto the target stack. TGT_STACK := $$(call PUSH,$${TGT_STACK},$${TGT}) ifneq "$$(strip $${SOURCES})" "" # This makefile builds one or more objects from source. Validate the # specified sources against the supported source file types. BAD_SRCS := $$(strip $$(filter-out $${ALL_SRC_EXTS},$${SOURCES})) ifneq "$${BAD_SRCS}" "" $$(error Unsupported source file(s) found in ${1} [$${BAD_SRCS}]) endif # Qualify and canonicalize paths. SOURCES := $$(call QUALIFY_PATH,$${DIR},$${SOURCES}) SOURCES := $$(call CANONICAL_PATH,$${SOURCES}) SRC_INCDIRS := $$(call QUALIFY_PATH,$${DIR},$${SRC_INCDIRS}) SRC_INCDIRS := $$(call CANONICAL_PATH,$${SRC_INCDIRS}) # Save the list of source files for this target. $${TGT}_SOURCES += $${SOURCES} # Convert the source file names to their corresponding object file # names. OBJS := $$(addprefix $${BUILD_DIR}/$$(call CANONICAL_PATH,$${TGT})/,\ $$(addsuffix .o,$$(basename $${SOURCES}))) # Add the objects to the current target's list of objects, and create # target-specific variables for the objects based on any source # variables that were defined. $${TGT}_OBJS += $${OBJS} $${TGT}_DEPS += $${OBJS:%.o=%.P} $${OBJS}: SRC_CFLAGS := $${$${TGT}_CFLAGS} $${SRC_CFLAGS} $${OBJS}: SRC_CXXFLAGS := $${$${TGT}_CXXFLAGS} $${SRC_CXXFLAGS} $${OBJS}: SRC_DEFS := $$(addprefix -D,$${$${TGT}_DEFS} $${SRC_DEFS}) $${OBJS}: SRC_INCDIRS := $$(addprefix -I,\ $${$${TGT}_INCDIRS} $${SRC_INCDIRS}) endif ifneq "$$(strip $${SUBMAKEFILES})" "" # This makefile has submakefiles. Recursively include them. $$(foreach MK,$${SUBMAKEFILES},\ $$(eval $$(call INCLUDE_SUBMAKEFILE,\ $$(call CANONICAL_PATH,\ $$(call QUALIFY_PATH,$${DIR},$${MK}))))) endif # Reset the "current" target to it's previous value. TGT_STACK := $$(call POP,$${TGT_STACK}) TGT := $$(call PEEK,$${TGT_STACK}) # Reset the "current" directory to it's previous value. DIR_STACK := $$(call POP,$${DIR_STACK}) DIR := $$(call PEEK,$${DIR_STACK}) endef # MIN - Parameterized "function" that results in the minimum lexical value of # the two values given. define MIN $(firstword $(sort ${1} ${2})) endef # PEEK - Parameterized "function" that results in the value at the top of the # specified colon-delimited stack. define PEEK $(lastword $(subst :, ,${1})) endef # POP - Parameterized "function" that pops the top value off of the specified # colon-delimited stack, and results in the new value of the stack. Note that # the popped value cannot be obtained using this function; use peek for that. define POP ${1:%:$(lastword $(subst :, ,${1}))=%} endef # PUSH - Parameterized "function" that pushes a value onto the specified colon- # delimited stack, and results in the new value of the stack. define PUSH ${2:%=${1}:%} endef # QUALIFY_PATH - Given a "root" directory and one or more paths, qualifies the # paths using the "root" directory (i.e. appends the root directory name to # the paths) except for paths that are absolute. define QUALIFY_PATH $(addprefix ${1}/,$(filter-out /%,${2})) $(filter /%,${2}) endef ############################################################################### # # Start of Makefile Evaluation # ############################################################################### # Older versions of GNU Make lack capabilities needed by boilermake. # With older versions, "make" may simply output "nothing to do", likely leading # to confusion. To avoid this, check the version of GNU make up-front and # inform the user if their version of make doesn't meet the minimum required. MIN_MAKE_VERSION := 3.81 MIN_MAKE_VER_MSG := boilermake requires GNU Make ${MIN_MAKE_VERSION} or greater ifeq "${MAKE_VERSION}" "" $(info GNU Make not detected) $(error ${MIN_MAKE_VER_MSG}) endif ifneq "${MIN_MAKE_VERSION}" "$(call MIN,${MIN_MAKE_VERSION},${MAKE_VERSION})" $(info This is GNU Make version ${MAKE_VERSION}) $(error ${MIN_MAKE_VER_MSG}) endif # Define the source file extensions that we know how to handle. C_SRC_EXTS := %.c CXX_SRC_EXTS := %.C %.cc %.cp %.cpp %.CPP %.cxx %.c++ JAVA_EXTS := %.jar %.tar ALL_SRC_EXTS := ${C_SRC_EXTS} ${CXX_SRC_EXTS} ${JAVA_EXTS} # Initialize global variables. ALL_TGTS := DEFS := DIR_STACK := INCDIRS := TGT_STACK := # Discover our OS and architecture. These are used to set the BUILD_DIR and TARGET_DIR to # something more useful than 'build' and '.'. OSTYPE := $(shell echo `uname`) OSVERSION := $(shell echo `uname -r`) MACHINETYPE := $(shell echo `uname -m`) ifeq (${MACHINETYPE}, x86_64) MACHINETYPE = amd64 endif ifeq (${MACHINETYPE}, Power Macintosh) MACHINETYPE = ppc endif ifeq (${OSTYPE}, SunOS) MACHINETYPE = ${shell echo `uname -p`} ifeq (${MACHINETYPE}, sparc) ifeq (${shell /usr/bin/isainfo -b}, 64) MACHINETYPE = sparc64 else MACHINETYPE = sparc32 endif endif endif # Set compiler and flags based on discovered hardware ifeq (${OSTYPE}, Linux) CC ?= gcc CXX ?= g++ CXXFLAGS := -pthread -fPIC -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare # -Wformat -Wall -Wextra LDFLAGS := -pthread -lm CXXFLAGS += -D_GLIBCXX_PARALLEL -fopenmp LDFLAGS += -fopenmp CXXFLAGS += -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat ifeq ($(BUILDPROFILE), 1) CXXFLAGS += LDFLAGS += -pg else CXXFLAGS += -D_GLIBCXX_PARALLEL LDFLAGS += -D_GLIBCXX_PARALLEL endif ifeq ($(BUILDDEBUG), 1) CXXFLAGS += -g else ifeq ($(BUILDPROFILE), 1) CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions else CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer endif endif endif ifeq (${OSTYPE}, Darwin) CLANG = $(shell echo `${CXX} --version 2>&1 | grep -c clang`) CC ?= gcc CXX ?= g++ ifeq ($(CLANG), 0) CXXFLAGS := -pthread -fopenmp -fPIC -m64 -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat LDFLAGS := -pthread -fopenmp -lm else CXXFLAGS := -DBROKEN_CLANG_OpenMP -pthread -fPIC -m64 -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat LDFLAGS := -pthread -lm endif ifeq ($(BUILDPROFILE), 1) CXXFLAGS += LDFLAGS += -pg else CXXFLAGS += -D_GLIBCXX_PARALLEL LDFLAGS += -D_GLIBCXX_PARALLEL endif CXXFLAGS += LDFLAGS += ifeq ($(BUILDDEBUG), 1) CXXFLAGS += -g # -fno-omit-frame-pointer else ifeq ($(BUILDPROFILE), 1) CXXFLAGS += -g3 -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fno-omit-frame-pointer else CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer endif endif endif ifeq (${OSTYPE}, FreeBSD) ifeq (${MACHINETYPE}, amd64) CC ?= gcc48 CXX ?= g++48 CXXFLAGS := -pthread -fPIC -Wall -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat -Wextra LDFLAGS := -pthread -lm CXXFLAGS += -fopenmp LDFLAGS += -fopenmp -rpath /usr/local/lib/gcc48 # Google Performance Tools malloc and heapchecker (HEAPCHECK=normal) #CXXFLAGS += #LDFLAGS += -ltcmalloc # Google Performance Tools cpu profiler (CPUPROFILE=/path) #CXXFLAGS += #LDFLAGS += -lprofiler ifeq ($(BUILDPROFILE), 1) CXXFLAGS += LDFLAGS += -pg else CXXFLAGS += -D_GLIBCXX_PARALLEL LDFLAGS += -D_GLIBCXX_PARALLEL endif CXXFLAGS += -I/usr/local/include LDFLAGS += -L/usr/local/lib # callgrind #CXXFLAGS += -g3 -Wa,--gstabs -save-temps #CXXFLAGS += -DLIBUNWIND -I/usr/local/include #LDFLAGS += -L/usr/local/lib -lunwind -lunwind-x86_64 ifeq ($(BUILDDEBUG), 1) CXXFLAGS += -g else ifeq ($(BUILDPROFILE), 1) CXXFLAGS += -g3 -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fno-omit-frame-pointer else CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer endif endif endif endif ifeq (${OSTYPE}, FreeBSD) ifeq (${MACHINETYPE}, arm) CC ?= gcc48 CXX ?= g++48 CXXFLAGS := -pthread -fPIC -Wall -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat -Wextra LDFLAGS := -pthread -lm CXXFLAGS += -D_GLIBCXX_PARALLEL -fopenmp LDFLAGS += -D_GLIBCXX_PARALLEL -fopenmp -rpath /usr/local/lib/gcc48 CXXFLAGS += -I/usr/local/include -O3 -funroll-loops -fomit-frame-pointer LDFLAGS += -L/usr/local/lib endif endif ifneq (,$(findstring CYGWIN, ${OSTYPE})) CC ?= gcc CXX ?= g++ CXXFLAGS := -pthread -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare LDFLAGS := -pthread -lm CXXFLAGS += -fopenmp LDFLAGS += -fopenmp CXXFLAGS += -Wall -Wextra -Wno-write-strings -Wno-unused -Wno-char-subscripts -Wno-sign-compare -Wformat ifeq ($(BUILDPROFILE), 1) CXXFLAGS += LDFLAGS += -pg else CXXFLAGS += LDFLAGS += endif ifeq ($(BUILDDEBUG), 1) CXXFLAGS += -g else ifeq ($(BUILDPROFILE), 1) CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions else CXXFLAGS += -O4 -funroll-loops -fexpensive-optimizations -finline-functions -fomit-frame-pointer endif endif endif # Include the main user-supplied submakefile. This also recursively includes # all other user-supplied submakefiles. $(eval $(call INCLUDE_SUBMAKEFILE,main.mk)) # Perform post-processing on global variables as needed. DEFS := $(addprefix -D,${DEFS}) INCDIRS := $(addprefix -I,$(call CANONICAL_PATH,${INCDIRS})) # Define the "all" target (which simply builds all user-defined targets) as the # default goal. .PHONY: all all: UPDATE_VERSION MAKE_DIRS \ $(addprefix ${TARGET_DIR}/,${ALL_TGTS}) \ ${TARGET_DIR}/canu \ ${TARGET_DIR}/canu.defaults \ ${TARGET_DIR}/lib/canu/lib \ ${TARGET_DIR}/lib/canu/Consensus.pm \ ${TARGET_DIR}/lib/canu/CorrectReads.pm \ ${TARGET_DIR}/lib/canu/Configure.pm \ ${TARGET_DIR}/lib/canu/Defaults.pm \ ${TARGET_DIR}/lib/canu/ErrorEstimate.pm \ ${TARGET_DIR}/lib/canu/Execution.pm \ ${TARGET_DIR}/lib/canu/Gatekeeper.pm \ ${TARGET_DIR}/lib/canu/Grid.pm \ ${TARGET_DIR}/lib/canu/Grid_LSF.pm \ ${TARGET_DIR}/lib/canu/Grid_PBSTorque.pm \ ${TARGET_DIR}/lib/canu/Grid_SGE.pm \ ${TARGET_DIR}/lib/canu/Grid_Slurm.pm \ ${TARGET_DIR}/lib/canu/HTML.pm \ ${TARGET_DIR}/lib/canu/Meryl.pm \ ${TARGET_DIR}/lib/canu/Output.pm \ ${TARGET_DIR}/lib/canu/OverlapBasedTrimming.pm \ ${TARGET_DIR}/lib/canu/OverlapErrorAdjustment.pm \ ${TARGET_DIR}/lib/canu/OverlapInCore.pm \ ${TARGET_DIR}/lib/canu/OverlapMhap.pm \ ${TARGET_DIR}/lib/canu/OverlapMMap.pm \ ${TARGET_DIR}/lib/canu/OverlapStore.pm \ ${TARGET_DIR}/lib/canu/Unitig.pm @echo "" @echo "Success!" @echo "canu installed in ${TARGET_DIR}/canu" # Add a new target rule for each user-defined target. $(foreach TGT,${ALL_TGTS},\ $(eval $(call ADD_TARGET_RULE,${TGT}))) # Add pattern rule(s) for creating compiled object code from C source. $(foreach TGT,${ALL_TGTS},\ $(foreach EXT,${C_SRC_EXTS},\ $(eval $(call ADD_OBJECT_RULE,${BUILD_DIR}/$(call CANONICAL_PATH,${TGT}),\ ${EXT},$${COMPILE_C_CMDS})))) # Add pattern rule(s) for creating compiled object code from C++ source. $(foreach TGT,${ALL_TGTS},\ $(foreach EXT,${CXX_SRC_EXTS},\ $(eval $(call ADD_OBJECT_RULE,${BUILD_DIR}/$(call CANONICAL_PATH,${TGT}),\ ${EXT},$${COMPILE_CXX_CMDS})))) # Add "clean" rules to remove all build-generated files. .PHONY: clean $(foreach TGT,${ALL_TGTS},\ $(eval $(call ADD_CLEAN_RULE,${TGT}))) # Include generated rules that define additional (header) dependencies. $(foreach TGT,${ALL_TGTS},\ $(eval -include ${${TGT}_DEPS})) # A fake target, to regenerate the canu_version.H file on every build. .PHONY: UPDATE_VERSION UPDATE_VERSION: @./canu_version_update.pl AS_global.C: UPDATE_VERSION # A fake target, to make the directory for the canu perl modules. .PHONY: MAKE_DIRS MAKE_DIRS: @if [ ! -e ${TARGET_DIR}/lib/canu ] ; then mkdir -p ${TARGET_DIR}/lib/canu ; fi ${TARGET_DIR}/canu: pipelines/canu.pl cp -pf pipelines/canu.pl ${TARGET_DIR}/canu chmod +x ${TARGET_DIR}/canu ${TARGET_DIR}/canu.defaults: echo > ${TARGET_DIR}/canu.defaults "# Add site specific options (for setting up Grid or limiting memory/threads) here." chmod -x ${TARGET_DIR}/canu.defaults ${TARGET_DIR}/lib/canu/lib: pipelines/install-perl-libraries.sh cd pipelines && sh install-perl-libraries.sh ${TARGET_DIR}/lib/canu ${TARGET_DIR}/lib/canu/Consensus.pm: pipelines/canu/Consensus.pm cp -pf pipelines/canu/Consensus.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/CorrectReads.pm: pipelines/canu/CorrectReads.pm cp -pf pipelines/canu/CorrectReads.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Configure.pm: pipelines/canu/Configure.pm cp -pf pipelines/canu/Configure.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Defaults.pm: pipelines/canu/Defaults.pm cp -pf pipelines/canu/Defaults.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/ErrorEstimate.pm: pipelines/canu/ErrorEstimate.pm cp -pf pipelines/canu/ErrorEstimate.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Execution.pm: pipelines/canu/Execution.pm cp -pf pipelines/canu/Execution.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Gatekeeper.pm: pipelines/canu/Gatekeeper.pm cp -pf pipelines/canu/Gatekeeper.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Grid.pm: pipelines/canu/Grid.pm cp -pf pipelines/canu/Grid.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Grid_LSF.pm: pipelines/canu/Grid_LSF.pm cp -pf pipelines/canu/Grid_LSF.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Grid_PBSTorque.pm: pipelines/canu/Grid_PBSTorque.pm cp -pf pipelines/canu/Grid_PBSTorque.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Grid_SGE.pm: pipelines/canu/Grid_SGE.pm cp -pf pipelines/canu/Grid_SGE.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Grid_Slurm.pm: pipelines/canu/Grid_Slurm.pm cp -pf pipelines/canu/Grid_Slurm.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/HTML.pm: pipelines/canu/HTML.pm cp -pf pipelines/canu/HTML.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Meryl.pm: pipelines/canu/Meryl.pm cp -pf pipelines/canu/Meryl.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Output.pm: pipelines/canu/Output.pm cp -pf pipelines/canu/Output.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/OverlapBasedTrimming.pm: pipelines/canu/OverlapBasedTrimming.pm cp -pf pipelines/canu/OverlapBasedTrimming.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/OverlapErrorAdjustment.pm: pipelines/canu/OverlapErrorAdjustment.pm cp -pf pipelines/canu/OverlapErrorAdjustment.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/OverlapInCore.pm: pipelines/canu/OverlapInCore.pm cp -pf pipelines/canu/OverlapInCore.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/OverlapMhap.pm: pipelines/canu/OverlapMhap.pm cp -pf pipelines/canu/OverlapMhap.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/OverlapMMap.pm: pipelines/canu/OverlapMMap.pm cp -pf pipelines/canu/OverlapMMap.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/OverlapStore.pm: pipelines/canu/OverlapStore.pm cp -pf pipelines/canu/OverlapStore.pm ${TARGET_DIR}/lib/canu/ ${TARGET_DIR}/lib/canu/Unitig.pm: pipelines/canu/Unitig.pm cp -pf pipelines/canu/Unitig.pm ${TARGET_DIR}/lib/canu/ # Makefile processed. Report that we're starting the build. ${info Building for '${OSTYPE}' '${OSVERSION}' as '${MACHINETYPE}' into '${DESTDIR}${PREFIX}/$(OSTYPE)-$(MACHINETYPE)/{bin,obj}'} #${info Using LD_RUN_PATH '${LD_RUN_PATH}'} canu-1.3/src/bogart-analysis/000077500000000000000000000000001272335422300161745ustar00rootroot00000000000000canu-1.3/src/bogart-analysis/analyze-mapped-unitigs-for-joins.pl000066400000000000000000000153301272335422300250260ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $alignedReads = shift @ARGV; die "Can't find blasr-coords aligned reads in '$alignedReads'\n" if (! -e "$alignedReads"); my $b1last = 0; my $e1last = 0; my $n1last = ""; my $f1last = "fwd"; my $b2last = 0; my $e2last = 0; my $n2last = ""; my $f2last = "fwd"; my $nBubble = 0; my $nSpan = 0; my $nGap = 0; my $nGapEv = 0; my $nAbut = 0; my $nOvl = 0; my $nOvlEv = 0; # Load the name to iid map. my %NAMtoIID; my %NAMtoUID; my %UIDtoIID; my %IIDtoDEG; my %IIDtoUTG; open(F, "< test.gkpStore.fastqUIDmap") or die "Failed to open 'test.gkpStore.fastqUIDmap'\n"; while () { my @v = split '\s+', $_; if (scalar(@v) == 3) { $NAMtoIID{$v[2]} = $v[1]; $NAMtoUID{$v[2]} = $v[0]; $UIDtoIID{$v[0]} = $v[1]; } else { $NAMtoIID{$v[2]} = $v[1]; $NAMtoUID{$v[2]} = $v[0]; $UIDtoIID{$v[0]} = $v[1]; $NAMtoIID{$v[5]} = $v[4]; $NAMtoUID{$v[5]} = $v[3]; $UIDtoIID{$v[3]} = $v[4]; } } close(F); print STDERR "Loaded ", scalar(keys %NAMtoIID), " read names.\n"; open(F, "< 9-terminator/test.posmap.frgdeg") or die "Failed to open '9-terminator/test.posmap.frgdeg'\n"; while () { my @v = split '\s+', $_; my $i = $UIDtoIID{$v[0]}; die "$_" if (!defined($i)); $IIDtoDEG{$i} = $v[1]; } close(F); print STDERR "Loaded ", scalar(keys %IIDtoDEG), " degenerate unitig reads.\n"; open(F, "< 9-terminator/test.posmap.frgutg") or die "Failed to open '9-terminator/test.posmap.frgutg'\n"; while () { my @v = split '\s+', $_; my $i = $UIDtoIID{$v[0]}; die "$_" if (!defined($i)); $IIDtoUTG{$i} = $v[1]; } close(F); print STDERR "Loaded ", scalar(keys %IIDtoUTG), " unitig reads.\n"; # UTG.coords is standard nucmer output. open(F, "< 9-terminator/UTG.coords") or die "Failed to open '9-terminator/UTG.coords'\n"; $_ = ; $_ = ; $_ = ; $_ = ; $_ = ; while () { s/^\s+//; s/\s+$//; my @v = split '\s+', $_; my $b1 = $v[0]; # Assembly coords my $e1 = $v[1]; my $b2 = $v[3]; # Read coords my $e2 = $v[4]; my $f1 = ($b1 < $e1) ? "fwd" : "rev"; my $f2 = ($b2 < $e2) ? "fwd" : "rev"; ($b1,$e1) = ($e1,$b1) if ($b1 > $e1); # Doesn't seem to occur. ($b2,$e2) = ($e2,$b2) if ($b2 > $e2); # Definitely does occur. #my $l1 = $v[6]; #my $l2 = $v[7]; my $id = $v[9]; my $n1 = $v[11]; my $n2 = $v[12]; if (($b1last <= $b1) && ($e1 <= $e1last)) { $nBubble++; #print STDERR "BUBBLE $n2 ($b1,$e1) in $n2last ($b1last,$e1last)\n"; next; } $nSpan++; goto bail if ($n1last ne $n1); #print STDERR "last $b1last,$e1last curr $b1,$e1\n"; die "coords not sorted by increasing assembly start!\n" if ($b1last > $b1); $nGap++ if ($e1last < $b1); $nAbut++ if ($e1last == $b1); $nOvl++ if ($e1last > $b1); my $minOvlSpan = 150; if ($e1last < $b1) { my $ge = $b1; my $gb = $e1last; my $gap = $ge - $gb; my $span = 0; open(C, "< $alignedReads") or die "Failed to open '$alignedReads'\n"; while () { chomp; my @v = split '\s+', $_; next if ($v[9] ne $n1); # STOP changing the read name, blasr! if ($v[10] =~ m/^(.*\d+_\d+)\/\d+_\d+/) { $v[10] = $1; } my $iid = $NAMtoIID{$v[10]}; my $utg = $IIDtoUTG{$iid}; my $deg = $IIDtoDEG{$iid}; my $ann; #next if (defined($utg) || defined($deg)); if (defined($utg)) { $ann = "utg $utg"; } elsif (defined($deg)) { $ann = "deg $deg"; } else { $ann = "singleton "; } if (($v[0] + $minOvlSpan < $ge) && ($gb < $v[1] - $minOvlSpan)) { #print "$ann iid $iid -- $_\n"; $span++; } } close(C); if ($span > 0) { $nGapEv++; print "GAP $n1 $n2last ($b1last,$e1last,$f2last) -- $gap -- $n2 ($b1,$e1,$f2) -- SPAN $span\n"; print "\n"; print "\n"; } } if ($e1last == $b1) { } if ($e1last > $b1) { my $ge = $e1last; my $gb = $b1; my $ovl = $ge - $gb; my $span = 0; open(C, "< $alignedReads") or die "Failed to open '$alignedReads'\n"; while () { chomp; my @v = split '\s+', $_; next if ($v[9] ne $n1); if ($v[10] =~ m/^(.*\d+_\d+)\/\d+_\d+/) { $v[10] = $1; } my $iid = $NAMtoIID{$v[10]}; my $utg = $IIDtoUTG{$iid}; my $deg = $IIDtoDEG{$iid}; my $ann; #next if (defined($utg) || defined($deg)); if (defined($utg)) { $ann = "utg $utg"; } elsif (defined($deg)) { $ann = "deg $deg"; } else { $ann = "singleton "; } if (($v[0] + $minOvlSpan < $ge) && ($gb < $v[1] - $minOvlSpan)) { print "$ann iid $iid -- $_\n"; $span++; } } close(C); if ($span > 0) { $nOvlEv++; print "OVL $n1 $n2last ($b1last,$e1last,$f2last) -- $ovl -- $n2 ($b1,$e1,$f2) -- SPAN $span\n"; print "\n"; print "\n"; } } bail: ($b1last,$e1last,$n1last,$f1last) = ($b1,$e1,$n1,$f1); ($b2last,$e2last,$n2last,$f2last) = ($b2,$e2,$n2,$f2); } close(F); print STDERR "nBubble $nBubble\n"; print STDERR "nSpan $nSpan\n"; print STDERR "nGap $nGap\n"; print STDERR "nGap $nGapEv with evidence\n"; print STDERR "nAbut $nAbut\n"; print STDERR "nOvl $nOvl\n"; print STDERR "nOvl $nOvlEv with evidence\n"; canu-1.3/src/bogart-analysis/analyze-nucmer-gaps.pl000066400000000000000000000153301272335422300224150ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $alignedReads = shift @ARGV; die "Can't find blasr-coords aligned reads in '$alignedReads'\n" if (! -e "$alignedReads"); my $b1last = 0; my $e1last = 0; my $n1last = ""; my $f1last = "fwd"; my $b2last = 0; my $e2last = 0; my $n2last = ""; my $f2last = "fwd"; my $nBubble = 0; my $nSpan = 0; my $nGap = 0; my $nGapEv = 0; my $nAbut = 0; my $nOvl = 0; my $nOvlEv = 0; # Load the name to iid map. my %NAMtoIID; my %NAMtoUID; my %UIDtoIID; my %IIDtoDEG; my %IIDtoUTG; open(F, "< test.gkpStore.fastqUIDmap") or die "Failed to open 'test.gkpStore.fastqUIDmap'\n"; while () { my @v = split '\s+', $_; if (scalar(@v) == 3) { $NAMtoIID{$v[2]} = $v[1]; $NAMtoUID{$v[2]} = $v[0]; $UIDtoIID{$v[0]} = $v[1]; } else { $NAMtoIID{$v[2]} = $v[1]; $NAMtoUID{$v[2]} = $v[0]; $UIDtoIID{$v[0]} = $v[1]; $NAMtoIID{$v[5]} = $v[4]; $NAMtoUID{$v[5]} = $v[3]; $UIDtoIID{$v[3]} = $v[4]; } } close(F); print STDERR "Loaded ", scalar(keys %NAMtoIID), " read names.\n"; open(F, "< 9-terminator/test.posmap.frgdeg") or die "Failed to open '9-terminator/test.posmap.frgdeg'\n"; while () { my @v = split '\s+', $_; my $i = $UIDtoIID{$v[0]}; die "$_" if (!defined($i)); $IIDtoDEG{$i} = $v[1]; } close(F); print STDERR "Loaded ", scalar(keys %IIDtoDEG), " degenerate unitig reads.\n"; open(F, "< 9-terminator/test.posmap.frgutg") or die "Failed to open '9-terminator/test.posmap.frgdeg'\n"; while () { my @v = split '\s+', $_; my $i = $UIDtoIID{$v[0]}; die "$_" if (!defined($i)); $IIDtoUTG{$i} = $v[1]; } close(F); print STDERR "Loaded ", scalar(keys %IIDtoUTG), " unitig reads.\n"; # UTG.coords is standard nucmer output. open(F, "< 9-terminator/UTG.coords") or die "Failed to open '9-terminator/UTG.coords'\n"; $_ = ; $_ = ; $_ = ; $_ = ; $_ = ; while () { s/^\s+//; s/\s+$//; my @v = split '\s+', $_; my $b1 = $v[0]; # Assembly coords my $e1 = $v[1]; my $b2 = $v[3]; # Read coords my $e2 = $v[4]; my $f1 = ($b1 < $e1) ? "fwd" : "rev"; my $f2 = ($b2 < $e2) ? "fwd" : "rev"; ($b1,$e1) = ($e1,$b1) if ($b1 > $e1); # Doesn't seem to occur. ($b2,$e2) = ($e2,$b2) if ($b2 > $e2); # Definitely does occur. #my $l1 = $v[6]; #my $l2 = $v[7]; my $id = $v[9]; my $n1 = $v[11]; my $n2 = $v[12]; if (($b1last <= $b1) && ($e1 <= $e1last)) { $nBubble++; #print STDERR "BUBBLE $n2 ($b1,$e1) in $n2last ($b1last,$e1last)\n"; next; } $nSpan++; goto bail if ($n1last ne $n1); #print STDERR "last $b1last,$e1last curr $b1,$e1\n"; die "coords not sorted by increasing assembly start!\n" if ($b1last > $b1); $nGap++ if ($e1last < $b1); $nAbut++ if ($e1last == $b1); $nOvl++ if ($e1last > $b1); my $minOvlSpan = 150; if ($e1last < $b1) { my $ge = $b1; my $gb = $e1last; my $gap = $ge - $gb; my $span = 0; open(C, "< $alignedReads") or die "Failed to open '$alignedReads'\n"; while () { chomp; my @v = split '\s+', $_; next if ($v[9] ne $n1); # STOP changing the read name, blasr! if ($v[10] =~ m/^(.*\d+_\d+)\/\d+_\d+/) { $v[10] = $1; } my $iid = $NAMtoIID{$v[10]}; my $utg = $IIDtoUTG{$iid}; my $deg = $IIDtoDEG{$iid}; my $ann; #next if (defined($utg) || defined($deg)); if (defined($utg)) { $ann = "utg $utg"; } elsif (defined($deg)) { $ann = "deg $deg"; } else { $ann = "singleton "; } if (($v[0] + $minOvlSpan < $ge) && ($gb < $v[1] - $minOvlSpan)) { #print "$ann iid $iid -- $_\n"; $span++; } } close(C); if ($span > 0) { $nGapEv++; print "GAP $n1 $n2last ($b1last,$e1last,$f2last) -- $gap -- $n2 ($b1,$e1,$f2) -- SPAN $span\n"; print "\n"; print "\n"; } } if ($e1last == $b1) { } if ($e1last > $b1) { my $ge = $e1last; my $gb = $b1; my $ovl = $ge - $gb; my $span = 0; open(C, "< $alignedReads") or die "Failed to open '$alignedReads'\n"; while () { chomp; my @v = split '\s+', $_; next if ($v[9] ne $n1); if ($v[10] =~ m/^(.*\d+_\d+)\/\d+_\d+/) { $v[10] = $1; } my $iid = $NAMtoIID{$v[10]}; my $utg = $IIDtoUTG{$iid}; my $deg = $IIDtoDEG{$iid}; my $ann; #next if (defined($utg) || defined($deg)); if (defined($utg)) { $ann = "utg $utg"; } elsif (defined($deg)) { $ann = "deg $deg"; } else { $ann = "singleton "; } if (($v[0] + $minOvlSpan < $ge) && ($gb < $v[1] - $minOvlSpan)) { print "$ann iid $iid -- $_\n"; $span++; } } close(C); if ($span > 0) { $nOvlEv++; print "OVL $n1 $n2last ($b1last,$e1last,$f2last) -- $ovl -- $n2 ($b1,$e1,$f2) -- SPAN $span\n"; print "\n"; print "\n"; } } bail: ($b1last,$e1last,$n1last,$f1last) = ($b1,$e1,$n1,$f1); ($b2last,$e2last,$n2last,$f2last) = ($b2,$e2,$n2,$f2); } close(F); print STDERR "nBubble $nBubble\n"; print STDERR "nSpan $nSpan\n"; print STDERR "nGap $nGap\n"; print STDERR "nGap $nGapEv with evidence\n"; print STDERR "nAbut $nAbut\n"; print STDERR "nOvl $nOvl\n"; print STDERR "nOvl $nOvlEv with evidence\n"; canu-1.3/src/bogart-analysis/bogart-build-fasta.pl000066400000000000000000000111431272335422300222000ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $prefix; my $FILE = undef; my $REF = undef; my $withSGE = 0; my $withPart = 0; if (scalar(@ARGV) == 0) { die "usage: $0 [-sge] [-partition] PREFIX.tigStore REFERENCE.fasta\n"; } while (scalar(@ARGV) > 0) { if ($ARGV[0] eq "-sge") { $withSGE = 1; } elsif ($ARGV[0] eq "-partition") { $withPart = 1; } else { $FILE = $ARGV[0]; $REF = $ARGV[1]; shift @ARGV; } shift @ARGV; } die "Must supply 'PREFIX.tigStore' as first non-option argument.\n" if (!defined($FILE)); die "Must supply 'REFERENCE.fasta' as second non-option argument.\n" if (!defined($REF)); if ($FILE =~ m/^(.*).tigStore/) { $FILE = $1; } if ($FILE =~ m/^(.*)\.\d\d\d\./) { $prefix = $1; } else { die "Failed to find prefix in '$FILE', expecting PREFIX.###.tigStore.\n"; } die "Failed to find reference '$REF'\n" if (! -e $REF); my %detected1; my %detected2; if (! -e "$FILE.tigStore") { my $cmd; $cmd = "bogart \\n"; $cmd .= " -O ../$prefix.ovlStore \\n"; $cmd .= " -G ../$prefix.gkpStore \\n"; $cmd .= " -T ../$prefix.tigStore \\n"; $cmd .= " -B 75000 \\n"; $cmd .= " -e 0.03 \\n"; $cmd .= " -E 0 \\n"; $cmd .= " -b \\n"; $cmd .= " -m 7 \\n"; $cmd .= " -U \\n"; $cmd .= " -o $prefix \\n"; $cmd .= " -D all \\n"; $cmd .= " -d overlapQuality \\n"; print "$cmd\n"; die "Will not run command.\n"; } ######################################## if ($withPart) { my $cmd; $cmd .= "gatekeeper"; $cmd .= " -P $FILE.tigStore.partitioning ../$prefix.gkpStore"; system($cmd); print STDERR "Partitioned.\n"; exit; } ######################################## print STDERR "Running consensus\n"; open(R, "> $FILE.sge.sh") or die; print R "#!/bin/sh\n"; print R "idx=`printf %03d \$SGE_TASK_ID`\n"; print R "utgcns -f -g ../$prefix.gkpStore -t $FILE.tigStore 1 \$idx > $FILE.tigStore/seqDB.v002.p\${idx}.utg.err 2>&1\n"; close(R); open(R, "> $FILE.tmp.sh") or die; open(F, "ls $FILE.tigStore/seqDB.v001.p???.dat |"); while () { chomp; if (m/seqDB.v001.p(\d\d\d).dat$/) { my $idx = $1; if (! -e "$FILE.tigStore/seqDB.v002.p${idx}.dat") { print STDERR " $_\n"; print R "utgcns -f -g ../$prefix.gkpStore -t $FILE.tigStore 1 $idx > $FILE.tigStore/seqDB.v002.p${idx}.utg.err 2>&1 &\n"; } else { print STDERR " $_ finished.\n"; } } else { die "No '$_'\n"; } } close(F); print R "wait\n"; close(R); if ($withSGE) { #system("qsub -cwd -j y -o /dev/null -l memory=2g -t 1-9 -q servers.q,black.q $FILE.sge.sh"); exit; } else { system("sh $FILE.tmp.sh"); } #unlink "$FILE.tmp.sh"; #unlink "$FILE.sge.sh"; ######################################## print STDERR "Dumping fasta\n"; if (! -e "$FILE.fasta") { my $cmd; $cmd .= "tigStore"; $cmd .= " -g ../$prefix.gkpStore"; $cmd .= " -t $FILE.tigStore 2"; $cmd .= " -U -d consensus"; $cmd .= "> $FILE.fasta"; system($cmd); } print STDERR "Dumping layout\n"; if (! -e "$FILE.layout") { my $cmd; $cmd .= "tigStore"; $cmd .= " -g ../$prefix.gkpStore"; $cmd .= " -t $FILE.tigStore 2"; $cmd .= " -U -d layout"; $cmd .= "> $FILE.layout"; system($cmd); } ######################################## print STDERR "Running nucmer\n"; if (! -e "$FILE.delta") { my $cmd; $cmd .= "nucmer"; $cmd .= " --maxmatch --coords -p $FILE"; $cmd .= " $REF"; $cmd .= " $FILE.fasta"; system($cmd); } if (! -e "$FILE.png") { my $cmd; $cmd .= "mummerplot"; $cmd .= " --layout --filter -p $FILE -t png"; #$cmd .= " --layout -p $FILE -t png"; $cmd .= " $FILE.delta"; system($cmd); } canu-1.3/src/bogart-analysis/count-bubbles.pl000066400000000000000000000141131272335422300212750ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $FILE = "test.004.buildUnitigs"; $FILE = shift @ARGV if (scalar(@ARGV) > 0); if ((! -e "$FILE.tigStore") || (! -e "$FILE.fasta")) { die "Missing tigStore or fasta. Run build-fasta.pl\n"; } ######################################## print STDERR "Aligning unitigs to unitigs\n"; if (! -e "$FILE.allVall.sim4db" ) { my $cmd; $cmd .= "/work/kmer/FreeBSD-amd64/bin/snapper2 "; $cmd .= " -queries $FILE.fasta "; $cmd .= " -genomic $FILE.fasta "; $cmd .= " -mersize 16 "; $cmd .= " -minmatchidentity 94 "; $cmd .= " -minmatchcoverage 94 "; $cmd .= " -aligns "; $cmd .= " -verbose > $FILE.allVall.sim4db"; system($cmd); } print STDERR "Filtering alignments\n"; if (! -e "$FILE.bubbles.sim4db") { my $cmd; $cmd .= "/work/kmer/FreeBSD-amd64/bin/filterPolishes "; $cmd .= " -selfhits "; $cmd .= " -D "; $cmd .= " < $FILE.allVall.sim4db "; $cmd .= " > $FILE.bubbles.sim4db"; system($cmd); } ######################################## my %detected1; my %detected2; my $utg; my $ref; my $ide; my $cov; print STDERR "Reading bubble mapping\n"; open(F, "< $FILE.bubbles.sim4db") or die; while () { chomp; if (m/^edef=utg(\d+)/) { $utg = $1; } if (m/^ddef=utg(\d+)/) { $ref = $1; } if (m/^\d+\[(\d+)-\d+-\d+\]\s\d+\[\d+-\d+\]\s+<(\d+)-\d+-(\d+)-\w+-\w+>$/) { $ide = $3; $cov = $2; } if (m/^(\d+)-(\d+)\s\((\d+)-(\d+)\)\s/) { $cov = 100 * $cov / ($2 - $1); } if (m/^sim4end$/) { die if (!defined($utg)); die if (!defined($ref)); if (($ide >= 96) && ($cov >= 98)) { $detected1{"$utg"}++; $detected2{"$utg-$ref"}++; } #if ($cov < 100) { # print STDERR "LOW COV $utg - $ref -- $cov\n"; #} undef $utg; undef $ref; } } close(F); print STDERR "Found ", scalar(keys %detected2), " actual bubble instances, from ", scalar(keys %detected1), " unitigs.\n"; my $numUnique = 0; foreach my $k (keys %detected1) { if ($detected1{$k} == 1) { #print STDERR "UNIQUE $k\n"; $numUnique++; } } print STDERR "Found $numUnique uniquely placeable bubbles.\n"; # # Scan the log. Discover if we successfully merged a bubble. Report bubbles that we merged that # were not verified by mapping. Remove bubbles we are successful on from the list of alignments # (so we can next report what we failed on). # my $numMerged = 0; my $numMergedUnique = 0; my $numMergedMultiple = 0; my $numMergedExtra = 0; my $numMergedExtraMultiple = 0; open(F, "< test.005.bubblePopping.log") or die; while () { chomp; if (m/merged\sbubble\sunitig\s(\d+)\sinto\sunitig\s(\d+)$/) { $numMerged++; if (!exists($detected1{"$1"})) { #print STDERR "EXTRA $_ (no mapping at all)\n"; $numMergedExtra++; } elsif (!exists($detected2{"$1-$2"})) { #print STDERR "EXTRA $_ (no mapping for this pair out of $detected1{\"$1\"} alignments - possibly incorrectly placed!)\n"; $numMergedExtraMultiple++; } elsif ($detected1{"$1"} == 1) { $numMergedUnique++; } elsif ($detected1{"$1"} > 1) { #print STDERR "MULTI $_ ($detected1{\"$1\"} alignments - possibly incorrectly placed!)\n"; $numMergedMultiple++; } else { die; } delete $detected2{"$1-$2"}; delete $detected1{"$1"}; } } close(F); print STDERR "Merged $numMerged bubbles:\n"; print STDERR " $numMergedUnique were uniquely alignable.\n"; print STDERR " $numMergedMultiple were multiply alignable.\n"; print STDERR " $numMergedExtra were not alignable at all.\n"; print STDERR " $numMergedExtraMultiple were not alignable at the spot it was merged.\n"; # # Scan the layouts. For each un-popped bubble (BOG didn't pop, but it aligned to a unitig) count # the number of fragments in the unitig, and length of the tig. Report that and the number of # places the bubble mapped. # my $numMergedMissed = 0; my $numMergedMissedUnique = 0; my $id = 0; # Unitig ID we're looking at my $nf = 0; # Number of fragments my $ln = 0; # Length of the unitig open(F, "< $FILE.layout"); open(O, "> $FILE.bubblesMISSED"); while () { chomp; if (m/^unitig\s+(\d+)$/) { if ($nf > 0) { $numMergedMissed++ if ($detected1{$id} > 0); $numMergedMissedUnique++ if ($detected1{$id} == 1); print O "Missed merge unitig $id of length $ln with $nf frags -- mapped to $detected1{$id} places.\n"; } $id = $1; $nf = 0; $ln = 0; } if (($detected1{"$id"} > 0) && (m/^FRG.*position\s+(\d+)\s+(\d+)$/)) { $nf++; $ln = $1 if ($ln < $1); $ln = $2 if ($ln < $2); } } close(F); if ($nf > 0) { $numMergedMissed++ if ($detected1{$id} > 0); $numMergedMissedUnique++ if ($detected1{$id} == 1); print O "Missed merge unitig $id of length $ln with $nf frags -- mapped to $detected1{$id} places.\n"; } close(O); print STDERR "Failed to merge $numMergedMissed unitigs.\n"; print STDERR " $numMergedMissedUnique were uniquely placeable.\n"; canu-1.3/src/bogart-analysis/examine-mapping-ideal.pl000066400000000000000000000306041272335422300226670ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $TOOSHORT = 500; my $FILE = "test.004.buildUnitigs"; $FILE = shift @ARGV if (scalar(@ARGV) > 0); my $IDEAL = "../ideal/porphyromonas_gingivalis_w83.flx.3200bp.0900bp.FJRUAFO0.ideal.intervals"; if ((! -e "$FILE.tigStore") || (! -e "$FILE.fasta")) { die "Missing tigStore or fasta. Run build-fasta.pl\n"; } ######################################## if (! -e "$FILE.coords") { my $cmd; $cmd = "nucmer"; $cmd .= " --maxmatch --coords -p $FILE"; $cmd .= " /work/FRAGS/porphyromonas_gingivalis_w83/reference/AE015924.fasta"; $cmd .= " $FILE.fasta"; system($cmd); } ######################################## # Load the lengh of each sequence. my %length; if (-e "$FILE.fasta") { open(F, "< $FILE.fasta") or die; while () { if (m/^>(utg\d+)\s+len=(\d+)$/) { $length{$1} = $2; } } close(F); } else { print STDERR "No $FILE.fasta file found, cannot .....\n"; } ######################################## # Pass one, load the coords. Analyze anything with one match, counting # the amount of coverage. my %nucmer; if (-e "$FILE.coords") { open(F, "< $FILE.coords") or die; $_ = ; $_ = ; $_ = ; $_ = ; $_ = ; while () { s/^\s+//; s/\s$+//; my @v = split '\s+', $_; my $utgBgn = ($v[3] < $v[4]) ? $v[3] : $v[4]; my $utgEnd = ($v[3] < $v[4]) ? $v[4] : $v[3]; my $genBgn = ($v[0] < $v[1]) ? $v[0] : $v[1]; my $genEnd = ($v[0] < $v[1]) ? $v[1] : $v[0]; # Rearrange the coords so that bgn is ALWAYS less than end (we lose orientation). my $str = "$utgBgn\t$utgEnd\t$genBgn\t$genEnd"; if (exists($nucmer{$v[12]})) { $nucmer{$v[12]} .= "\n" . $str; } else { $nucmer{$v[12]} = $str; } } close(F); } else { die "No $FILE.coords??\n"; } ######################################## # For things with one match, report spurs. my $spur = 0; my $spurShort = 0; my $confirmed = 0; my $confirmedBP = 0; foreach my $utg (keys %nucmer) { my @m = split '\n', $nucmer{$utg}; if (scalar(@m) > 1) { @m = sort {$a <=> $b} @m; $nucmer{$utg} = join "\n", @m; next; } my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $nucmer{$utg}; my $len = $utgEnd - $utgBgn; die if ($len <= 0); my $per = $len / $length{$utg}; if ($per < 0.95) { if ($length{$utg} < $TOOSHORT) { $spurShort++; } else { print STDERR "SPUR $utg len $length{$utg} mapped $len ($per%) coords $nucmer{$utg}\n"; $spur++; } } else { $confirmed++; $confirmedBP += $length{$utg}; } delete $nucmer{$utg}; } print STDERR "FOUND ", scalar(keys %nucmer), " MATCHES, of which $spur are SPURs (and $spurShort are SHORT SPURS)\n"; print STDERR "FOUND $confirmed confirmed unitigs of total length $confirmedBP bp.\n"; ######################################## # Things with multiple matches are candidates for chimera. my $classFalseMateSplit = 0; my $lenFalseMateSplit = 0; my $classShort = 0; my $lenShort = 0; my $classDisconnected = 0; my $lenDisconnected = 0; my $classRepeat = 0; my $lenRepeat = 0; my $classRepeatEndWithRepeat = 0; my $lenRepeatEndWithRepeat = 0; my $classRepeat1Unique = 0; my $lenRepeat1Unique = 0; my $classRepeat2Unique = 0; my $lenRepeat2Unique = 0; my $classMultipleBlock = 0; my $lenMultipleBlock = 0; my $classUnknown = 0; my $lenUnknown = 0; foreach my $utg (keys %nucmer) { my @m = split '\n', $nucmer{$utg}; my $numBlocks = 0; my @bgn; my @end; my $min = 999999999; my $max = 0; #print STDOUT "$utg\n"; # Examine the matches, decide if the multiple matches are due to # unitig is a repeat # short repeats interior to the unitig # long repeats at the end # chimer my $numMatches = 0; my $isDisconnected = 0; my $verboseBuild = 0; print STDOUT "--------------------------------------------------------------------------------\n" if ($verboseBuild); print STDOUT "UNITIG $utg\n" if ($verboseBuild); foreach my $m (@m) { my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $m; $numMatches++; print STDOUT " $m" if ($verboseBuild); # Are we an exact match to the largest thing so far? if (($utgBgn == $min) && ($utgEnd == $max)) { print STDOUT " -- is exact\n" if ($verboseBuild); next; } # Search for things comtained or overlapping previous matches. This assumes that matches # are sorted by begin coord. We need to handle: # # ---------- and ----------- not -------- or ----- # ---- --------- ------ ---------- # # and, the ones we're more-or-less really looking for: # # -------- OR -------- # -------- ------- # This isolates out the first case above. We don't want to save it as a region, # because it is completely contained in a previous region. if ($utgEnd <= $max) { print STDOUT " -- is contained\n" if ($verboseBuild); next; } # If we aren't even intersecting, we've found the third case. $isDisconnected++ if (($max > 0) && ($max < $utgBgn)); # Finally, update the min/max extents. This must be last. if ($max == 0) { $min = $utgBgn; $max = $utgEnd; } if (($utgBgn - 5 <= $min) && ($max < $utgEnd)) { # max STRICTLY LESS than utgEnd! pop @end; push @end, $utgEnd; $max = $utgEnd; print STDOUT " -- extends ", $bgn[scalar(@bgn)-1], " ", $end[scalar(@end)-1], " ", scalar(@bgn), " ", scalar(@end), " $utgBgn $utgEnd\n" if ($verboseBuild); next; } if ($utgBgn < $min) { $min = $utgBgn; } if ($max < $utgEnd) { $max = $utgEnd; } push @bgn, $utgBgn; push @end, $utgEnd; $numBlocks++; print STDOUT "\n" if ($verboseBuild); } # One more pass through to count repeats. Above we just found the min/max extent of the # alignments on the unitig. Here we can count if alignments are interior to the extent, at the # end, or completely spanning it. # # This fails by counting interior disconnected chimer (e.g., -- -- --) as 'isInteriorRepeat' my $isMaxExact = 0; my $isInteriorRepeat = 0; my $isTerminalRepeat = 0; foreach my $m (@m) { my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $m; # New match covers the whole region. if (($utgBgn - 5 <= $min) && ($max <= $utgEnd + 5)) { $isMaxExact++; next; } # New match completely contained, near the extent. if (($utgBgn - 5 <= $min) || ($max <= $utgEnd + 5)) { $isTerminalRepeat++; next; } # Otherwise, the match must be interior. $isInteriorRepeat++; } print STDERR "UNDEF LENGTH $utg\n" if (!defined($length{$utg})); print STDERR "ZERO LENGTH $utg\n" if ($length{$utg} == 0); print STDERR "SHORT LENGTH $utg\n" if ($length{$utg} < 64); if (($numMatches == 2) && ($isDisconnected > 0) && ($isTerminalRepeat == 2)) { $classFalseMateSplit++; $lenFalseMateSplit += $length{$utg}; #print "$utg (len $length{$utg}) is a FALSE MATE SPLIT.\n"; next; } if ($isDisconnected > 0) { $classDisconnected++; $lenDisconnected += $length{$utg}; #print "$utg (len $length{$utg}) is DISCONNECTED.\n"; next; } if ($isMaxExact >= 2) { $classRepeat++; $lenRepeat += $length{$utg}; #print "$utg (len $length{$utg}) is a REPEAT.\n"; next; } if ($max < $TOOSHORT) { $classShort++; $lenShort += $length{$utg}; next; } if (($isMaxExact == 1) && ($isTerminalRepeat > 0) && ($isInteriorRepeat > 0)) { $classRepeatEndWithRepeat++; $lenRepeatEndWithRepeat += $length{$utg}; if (0) { print "----------\n"; print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n"; print "$nucmer{$utg}\n"; for (my $iii=0; $iii 1); next; } if (($isMaxExact == 1) && ($isTerminalRepeat == $numMatches - 1)) { $classRepeat1Unique++; $lenRepeat1Unique += $length{$utg}; if (0) { print "----------\n"; print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n"; print "$nucmer{$utg}\n"; for (my $iii=0; $iii 1) { $classMultipleBlock++; $lenMultipleBlock += $length{$utg}; if (0) { print "----------\n"; print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n"; print "$nucmer{$utg}\n"; for (my $iii=0; $iii 0); if ((! -e "$FILE.tigStore") || (! -e "$FILE.fasta")) { die "Missing tigStore or fasta. Run build-fasta.pl\n"; } ######################################## if (! -e "$FILE.coords") { my $cmd; $cmd = "nucmer"; $cmd .= " --maxmatch --coords -p $FILE"; $cmd .= " /work/FRAGS/porphyromonas_gingivalis_w83/reference/AE015924.fasta"; $cmd .= " $FILE.fasta"; system($cmd); } ######################################## # Load the lengh of each sequence. my %length; if (-e "$FILE.fasta") { open(F, "< $FILE.fasta") or die; while () { if (m/^>(utg\d+)\s+len=(\d+)$/) { $length{$1} = $2; } } close(F); } else { print STDERR "No $FILE.fasta file found, cannot .....\n"; } ######################################## # Pass one, load the coords. Analyze anything with one match, counting # the amount of coverage. my %nucmer; if (-e "$FILE.coords") { open(F, "< $FILE.coords") or die; $_ = ; $_ = ; $_ = ; $_ = ; $_ = ; while () { s/^\s+//; s/\s$+//; my @v = split '\s+', $_; my $utgBgn = ($v[3] < $v[4]) ? $v[3] : $v[4]; my $utgEnd = ($v[3] < $v[4]) ? $v[4] : $v[3]; my $genBgn = ($v[0] < $v[1]) ? $v[0] : $v[1]; my $genEnd = ($v[0] < $v[1]) ? $v[1] : $v[0]; # Rearrange the coords so that bgn is ALWAYS less than end (we lose orientation). my $str = "$utgBgn\t$utgEnd\t$genBgn\t$genEnd"; if (exists($nucmer{$v[12]})) { $nucmer{$v[12]} .= "\n" . $str; } else { $nucmer{$v[12]} = $str; } } close(F); } else { die "No $FILE.coords??\n"; } ######################################## # For things with one match, report spurs. my $spur = 0; my $spurShort = 0; my $confirmed = 0; my $confirmedBP = 0; foreach my $utg (keys %nucmer) { my @m = split '\n', $nucmer{$utg}; if (scalar(@m) > 1) { @m = sort {$a <=> $b} @m; $nucmer{$utg} = join "\n", @m; next; } my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $nucmer{$utg}; my $len = $utgEnd - $utgBgn; die if ($len <= 0); my $per = $len / $length{$utg}; if ($per < 0.95) { if ($length{$utg} < $TOOSHORT) { $spurShort++; } else { print STDERR "SPUR $utg len $length{$utg} mapped $len ($per%) coords $nucmer{$utg}\n"; $spur++; } } else { $confirmed++; $confirmedBP += $length{$utg}; } delete $nucmer{$utg}; } print STDERR "FOUND ", scalar(keys %nucmer), " MATCHES, of which $spur are SPURs (and $spurShort are SHORT SPURS)\n"; print STDERR "FOUND $confirmed confirmed unitigs of total length $confirmedBP bp.\n"; ######################################## # Things with multiple matches are candidates for chimera. my $classFalseMateSplit = 0; my $lenFalseMateSplit = 0; my $classShort = 0; my $lenShort = 0; my $classDisconnected = 0; my $lenDisconnected = 0; my $classRepeat = 0; my $lenRepeat = 0; my $classRepeatEndWithRepeat = 0; my $lenRepeatEndWithRepeat = 0; my $classRepeat1Unique = 0; my $lenRepeat1Unique = 0; my $classRepeat2Unique = 0; my $lenRepeat2Unique = 0; my $classMultipleBlock = 0; my $lenMultipleBlock = 0; my $classUnknown = 0; my $lenUnknown = 0; foreach my $utg (keys %nucmer) { my @m = split '\n', $nucmer{$utg}; my $numBlocks = 0; my @bgn; my @end; my $min = 999999999; my $max = 0; #print STDOUT "$utg\n"; # Examine the matches, decide if the multiple matches are due to # unitig is a repeat # short repeats interior to the unitig # long repeats at the end # chimer my $numMatches = 0; my $isDisconnected = 0; my $verboseBuild = 1; print STDOUT "--------------------------------------------------------------------------------\n" if ($verboseBuild); print STDOUT "UNITIG $utg\n" if ($verboseBuild); foreach my $m (@m) { my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $m; $numMatches++; print STDOUT " $m" if ($verboseBuild); # Are we an exact match to the largest thing so far? if (($utgBgn == $min) && ($utgEnd == $max)) { print STDOUT " -- is exact\n" if ($verboseBuild); next; } # Search for things comtained or overlapping previous matches. This assumes that matches # are sorted by begin coord. We need to handle: # # ---------- and ----------- not -------- or ----- # ---- --------- ------ ---------- # # and, the ones we're more-or-less really looking for: # # -------- OR -------- # -------- ------- # This isolates out the first case above. We don't want to save it as a region, # because it is completely contained in a previous region. if ($utgEnd <= $max) { print STDOUT " -- is contained in a region\n" if ($verboseBuild); next; } # If we aren't even intersecting, we've found the third case. my $dlabel = ""; if (($max > 0) && ($max < $utgBgn)) { my $dist = $utgBgn - $max; $dlabel = " DISCONNECT $dist"; $isDisconnected++ } # Finally, update the min/max extents. This must be last. if ($max == 0) { $min = $utgBgn; $max = $utgEnd; } if (($utgBgn - 5 <= $min) && ($max < $utgEnd)) { # max STRICTLY LESS than utgEnd! pop @end; push @end, $utgEnd; $max = $utgEnd; print STDOUT " -- extends a region\n" if ($verboseBuild); next; } if ($utgBgn < $min) { $min = $utgBgn; } if ($max < $utgEnd) { $max = $utgEnd; } push @bgn, $utgBgn; push @end, $utgEnd; $numBlocks++; print STDOUT " -- makes a new region$dlabel\n" if ($verboseBuild); } # One more pass through to count repeats. Above we just found the min/max extent of the # alignments on the unitig. Here we can count if alignments are interior to the extent, at the # end, or completely spanning it. # # This fails by counting interior disconnected chimer (e.g., -- -- --) as 'isInteriorRepeat' my $isMaxExact = 0; my $isInteriorRepeat = 0; my $isTerminalRepeat = 0; foreach my $m (@m) { my ($utgBgn, $utgEnd, $genBgn, $genEnd) = split '\s+', $m; # New match covers the whole region. if (($utgBgn - 5 <= $min) && ($max <= $utgEnd + 5)) { $isMaxExact++; next; } # New match completely contained, near the extent. if (($utgBgn - 5 <= $min) || ($max <= $utgEnd + 5)) { $isTerminalRepeat++; next; } # Otherwise, the match must be interior. $isInteriorRepeat++; } print STDERR "UNDEF LENGTH $utg\n" if (!defined($length{$utg})); print STDERR "ZERO LENGTH $utg\n" if ($length{$utg} == 0); print STDERR "SHORT LENGTH $utg\n" if ($length{$utg} < 64); if (($numMatches == 2) && ($isDisconnected > 0) && ($isTerminalRepeat == 2)) { $classFalseMateSplit++; $lenFalseMateSplit += $length{$utg}; print "$utg (len $length{$utg}) is a FALSE MATE SPLIT.\n" if ($verboseBuild); next; } if ($isDisconnected > 0) { $classDisconnected++; $lenDisconnected += $length{$utg}; print "$utg (len $length{$utg}) is DISCONNECTED.\n" if ($verboseBuild); next; } if ($isMaxExact >= 2) { $classRepeat++; $lenRepeat += $length{$utg}; print "$utg (len $length{$utg}) is a REPEAT.\n" if ($verboseBuild); next; } if ($max < $TOOSHORT) { $classShort++; $lenShort += $length{$utg}; next; } if (($isMaxExact == 1) && ($isTerminalRepeat > 0) && ($isInteriorRepeat > 0)) { $classRepeatEndWithRepeat++; $lenRepeatEndWithRepeat += $length{$utg}; if (0) { print "----------\n"; print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n"; print "$nucmer{$utg}\n"; for (my $iii=0; $iii 1); next; } if (($isMaxExact == 1) && ($isTerminalRepeat == $numMatches - 1)) { $classRepeat1Unique++; $lenRepeat1Unique += $length{$utg}; if (0) { print "----------\n"; print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n"; print "$nucmer{$utg}\n"; for (my $iii=0; $iii 1) { $classMultipleBlock++; $lenMultipleBlock += $length{$utg}; if (0) { print "----------\n"; print "$utg extent $min $max numBlocks $numBlocks numMatches $numMatches maxExact $isMaxExact interiorRepeat $isInteriorRepeat terminalRepeat $isTerminalRepeat disconnect $isDisconnected\n"; print "$nucmer{$utg}\n"; for (my $iii=0; $iii; my $len = ; my $cns = ; my $qlt = ; my $d1 = ; # cov stat my $d2 = ; # microhet my $d3 = ; # status my $d4 = ; # unique_rept my $d5 = ; # status my $nfrg = ; # num frags my $d7 = ; # num unitigs my $maxPos = 0; if ($utg =~ m/unitig\s+(\d+)$/) { $utg = $1; } else { die "Out of sync on '$utg'\n"; } if ($nfrg =~ m/data.num_frags\s+(\d+)$/) { $nfrg = $1; } else { die "Out of sync on '$nfrg'\n"; } my @frgs; for (my $nnnn=0; $nnnn < $nfrg; $nnnn++) { $_ = ; chomp; if (m/^FRG\stype\sR\sident\s+(\d+)\s+.*position\s+(\d+)\s+(\d+)$/) { push @frgs, "FRG\t$1\t$2\t$3"; $maxPos = ($maxPos < $2) ? $2 : $maxPos; $maxPos = ($maxPos < $3) ? $3 : $maxPos; } else { die "Nope: '$_'\n"; } } push @tigStore, "UTG\t$utg\t$nfrg\t$maxPos"; push @tigStore, @frgs; } close(F); if (defined($ARGV[0])) { push @reads, $ARGV[0]; } else { print STDERR "SCANNING tigStore for reads that span a unitig.\n"; my $chk; my $utg; my $nfrg = 0; my $tfrg = 0; my $maxPos; foreach my $l (@tigStore) { if ($nfrg == 0) { ($chk, $utg, $nfrg, $maxPos) = split '\s+', $l; die if ($chk ne "UTG"); next; } $nfrg--; $tfrg++; my ($chk, $iid, $bgn, $end) = split '\s+', $l; die if ($chk ne "FRG"); if ($bgn > $end) { ($bgn, $end) = ($end, $bgn); } if (($bgn == 0) && ($end == $maxPos)) { push @reads, $iid; } } print STDERR "Searching for placements for ", scalar(@reads), " reads.\n"; } foreach my $read (@reads) { my %ovlRead; my %ovlType; #print STDERR "Searching for placements for read $read in unitigs.\n"; print "\n"; # Find overlapping reads. open(F, "overlapStore -b $read -e $read -d $prefix.ovlStore |"); while () { s/^\s+//; s/\s+$//; my @v = split '\s+', $_; $ovlRead{$v[1]} = 1; $ovlType{$v[1]} = ''; if (($v[3] < 0) && ($v[4] < 0)) { $ovlType{$v[1]} = '5'; } elsif (($v[3] > 0) && ($v[4] > 0)) { $ovlType{$v[1]} = '3'; } elsif (($v[3] < 0) && ($v[4] > 0)) { $ovlType{$v[1]} = 'c'; # Contained } else { $ovlType{$v[1]} = 'C'; # Container } } close(F); #print STDERR "Searching for placements for read $read in unitigs, using ", scalar(keys %ovlRead), " overlapping reads.\n"; # Scan unitigs. For any unitig with more than zero overlapping reads present, report. my $chk; my $utg; my $nfrg = 0; my $cfrg = 0; my $maxPos; my $nmat; my %ntyp; my $self; foreach my $l (@tigStore) { if ($cfrg == 0) { ($chk, $utg, $nfrg, $maxPos) = split '\s+', $l; die if ($chk ne "UTG"); $cfrg = $nfrg; $nmat = 0; $ntyp{'5'} = 0; $ntyp{'3'} = 0; $ntyp{'C'} = 0; $ntyp{'c'} = 0; $self = 0; next; } $cfrg--; my ($chk, $iid, $bgn, $end) = split '\s+', $l; die if ($chk ne "FRG"); if ($bgn > $end) { ($bgn, $end) = ($end, $bgn); } if (exists($ovlRead{$iid})) { $nmat++; $ntyp{$ovlType{$iid}}++; #print "FRG\t$iid\tat\t$bgn\t$end\t$ovlType{$iid}\n"; } if ($iid == $read) { $self++; } if (($cfrg == 0) && ($self + $nmat > 0)) { if ($self) { print "read $read\tUNITIG $utg\tof length $maxPos with 5'=$ntyp{'5'} 3'=$ntyp{'3'} contained=$ntyp{'c'} container=$ntyp{'C'} overlapping reads out of $nfrg.\n"; } else { print "read $read\tunitig $utg\tof length $maxPos with 5'=$ntyp{'5'} 3'=$ntyp{'3'} contained=$ntyp{'c'} container=$ntyp{'C'} overlapping reads out of $nfrg.\n"; } } } close(F); } canu-1.3/src/bogart-analysis/show-false-best-edges-from-mapping.pl000066400000000000000000000045361272335422300252230ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $asm = shift @ARGV; # Reads: # $asm//best.edges # filtered-overlaps.true.ova # filtered-overlaps.false.ova # # Reports which best.edges are false my %true; my %false; open(F, "< filtered-overlaps.true.ova") or die; while () { s/^\s+//; s/\s+$//; my @v = split '\s+', $_; $true{"$v[0]-$v[1]"}++; $true{"$v[1]-$v[0]"}++; } close(F); open(F, "< filtered-overlaps.false.ova") or die; while () { s/^\s+//; s/\s+$//; my @v = split '\s+', $_; $false{"$v[0]-$v[1]"}++; $false{"$v[1]-$v[0]"}++; } close(F); my $true5 = 0; my $true3 = 0; my $false5 = 0; my $false3 = 0; my $novel5 = 0; my $novel3 = 0; open(F, "< $asm/best.edges") or die; while () { s/^\s+//; s/\s+$//; my @v = split '\s+', $_; my $p5 = "$v[0]-$v[2]"; my $p3 = "$v[0]-$v[4]"; if (exists($true{$p5})) { $true5++; } elsif (exists($false{$p5})) { print STDERR "FALSE 5 $_\n"; $false5++; } else { print STDERR "NOVEL 5 $_\n"; $novel5++; } if (exists($true{$p3})) { $true3++; } elsif (exists($false{$p3})) { print STDERR "FALSE 3 $_\n"; $false3++; } else { print STDERR "NOVEL 3 $_\n"; $novel3++; } } close(F); print "true5 $true5\n"; print "true3 $true3\n"; print "false5 $false5\n"; print "false3 $false3\n"; print "novel5 $novel5\n"; print "novel3 $novel3\n"; canu-1.3/src/bogart/000077500000000000000000000000001272335422300143535ustar00rootroot00000000000000canu-1.3/src/bogart/AS_BAT_BestOverlapGraph.C000066400000000000000000000751561272335422300207560ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_BestOverlapGraph.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2014-JAN-29 * are Copyright 2010-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-09 to 2015-AUG-14 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-MAR-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "intervalList.H" #include "stddev.H" void BestOverlapGraph::removeSuspicious(void) { uint32 fiLimit = FI->numFragments(); uint32 numThreads = omp_get_max_threads(); uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99; writeLog("BestOverlapGraph()-- removing suspicious reads from graph, with %d threads.\n", numThreads); #pragma omp parallel for schedule(dynamic, blockSize) for (uint32 fi=1; fi <= fiLimit; fi++) { uint32 no = 0; BAToverlap *ovl = OC->getOverlaps(fi, AS_MAX_EVALUE, no); bool verified = false; intervalList IL; uint32 fLen = FI->fragmentLength(fi); for (uint32 ii=0; (ii= 0) && (ovl[ii].b_hang >= 0)) // Right side dovetail IL.add(ovl[ii].a_hang, fLen - ovl[ii].a_hang); else if ((ovl[ii].a_hang >= 0) && (ovl[ii].b_hang <= 0)) // I contain the other IL.add(ovl[ii].a_hang, fLen - ovl[ii].a_hang - ovl[ii].b_hang); else if ((ovl[ii].a_hang <= 0) && (ovl[ii].b_hang >= 0)) // I am contained and thus now perfectly good! verified = true; else // Huh? Coding error. assert(0); } if (verified == false) { IL.merge(); verified = (IL.numberOfIntervals() == 1); } if (verified == false) { #pragma omp critical (suspInsert) _suspicious.insert(fi); } } writeLog("BestOverlapGraph()-- marked "F_U64" reads as suspicious.\n", _suspicious.size()); } void BestOverlapGraph::removeHighErrorBestEdges(void) { uint32 fiLimit = FI->numFragments(); uint32 numThreads = omp_get_max_threads(); uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99; stdDev edgeStats; // Find the overlap for every best edge. double *erates = new double [fiLimit + 1 + fiLimit + 1]; double *absdev = new double [fiLimit + 1 + fiLimit + 1]; uint32 eratesLen = 0; for (uint32 fi=1; fi <= fiLimit; fi++) { BestEdgeOverlap *b5 = getBestEdgeOverlap(fi, false); BestEdgeOverlap *b3 = getBestEdgeOverlap(fi, true); if (b5->fragId() != 0) edgeStats.insert(erates[eratesLen++] = b5->erate()); if (b3->fragId() != 0) edgeStats.insert(erates[eratesLen++] = b3->erate()); } _mean = edgeStats.mean(); _stddev = edgeStats.stddev(); writeLog("removeHighErrorBestEdges()-- with %u points - mean %f stddev %f -- would use overlaps below %f fraction error\n", edgeStats.size(), _mean, _stddev, _mean + _deviationGraph * _stddev); // Find the median and absolute deviations. sort(erates, erates+eratesLen); _median = erates[ eratesLen / 2 ]; for (uint32 ii=0; ii= 0.0); _mad = absdev[eratesLen/2]; delete [] absdev; delete [] erates; writeLog("removeHighErrorBestEdges()-- with %u points - median %f mad %f - would use overlaps below %f fraction error\n", edgeStats.size(), _median, _mad, _median + _deviationGraph * 1.4826 * _mad); // The real filtering is done on the next pass through findEdges(). Here, we just report statistics. uint32 noedge = 0; uint32 removed = 0; uint32 retained = 0; for (uint32 fi=1; fi <= fiLimit; fi++) { BestEdgeOverlap *b5 = getBestEdgeOverlap(fi, false); BestEdgeOverlap *b3 = getBestEdgeOverlap(fi, true); if (b5->fragId() == 0) noedge++; else if (b5->erate() > _mean + _deviationGraph * _stddev) removed++; else retained++; if (b3->fragId() == 0) noedge++; else if (b3->erate() > _mean + _deviationGraph * _stddev) removed++; else retained++; } writeLog("removeHighErrorBestEdges()-- %u ends have no best edge; %u ends are suspiciously high error; %u ends are acceptable.\n", noedge, removed, retained); } void BestOverlapGraph::removeLopsidedEdges(void) { uint32 fiLimit = FI->numFragments(); uint32 numThreads = omp_get_max_threads(); uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99; writeLog("BestOverlapGraph()-- removing suspicious edges from graph, with %d threads.\n", numThreads); uint32 nSuspicious = 0; uint32 nContained = 0; uint32 nSpur = 0; uint32 nMutual = 0; uint32 nAccepted = 0; uint32 nRejected = 0; #pragma omp parallel for schedule(dynamic, blockSize) for (uint32 fi=1; fi <= fiLimit; fi++) { BestEdgeOverlap *this5 = getBestEdgeOverlap(fi, false); BestEdgeOverlap *this3 = getBestEdgeOverlap(fi, true); // Ignore spurs and contains...and previously detected suspicious reads. The suspicious reads // do not have best edges back to them, and it's possible to find reads B where best edge A->B // exists, yet no best edge from B exists. if (isSuspicious(fi) == true) { #pragma omp atomic nSuspicious++; continue; } if (isContained(fi) == true) { #pragma omp atomic nContained++; continue; } if ((this5->fragId() == 0) || (this3->fragId() == 0)) { #pragma omp atomic nSpur++; continue; } // Find the overlap for this5 and this3. int32 this5ovlLen = FI->overlapLength(fi, this5->fragId(), this5->ahang(), this5->bhang()); int32 this3ovlLen = FI->overlapLength(fi, this3->fragId(), this3->ahang(), this3->bhang()); // Find the edges for our best overlaps. BestEdgeOverlap *that5 = getBestEdgeOverlap(this5->fragId(), this5->frag3p()); BestEdgeOverlap *that3 = getBestEdgeOverlap(this3->fragId(), this3->frag3p()); // If both point back to us, we're done. if ((that5->fragId() == fi) && (that5->frag3p() == false) && (that3->fragId() == fi) && (that3->frag3p() == true)) { #pragma omp atomic nMutual++; continue; } // If there is an overlap to something with no overlaps out of it, that's // a little suspicious. if ((that5->fragId() == 0) || (that3->fragId() == 0)) { writeLog("WARNING: read %u has overlap to spur - 3' to read %u back to %u - 5' to read %u back to %u\n", fi, this5->fragId(), that5->fragId(), this3->fragId(), that3->fragId()); #pragma omp critical (suspInsert) _suspicious.insert(fi); continue; } // Something doesn't agree. Find those overlaps... int32 that5ovlLen = FI->overlapLength(this5->fragId(), that5->fragId(), that5->ahang(), that5->bhang()); int32 that3ovlLen = FI->overlapLength(this3->fragId(), that3->fragId(), that3->ahang(), that3->bhang()); // ...and compare. double percDiff5 = 200.0 * abs(this5ovlLen - that5ovlLen) / (this5ovlLen + that5ovlLen); double percDiff3 = 200.0 * abs(this3ovlLen - that3ovlLen) / (this3ovlLen + that3ovlLen); if ((percDiff5 <= 5) && (percDiff3 <= 5)) { #if 0 writeLog("fi %8u -- %8u/%c' len %6u VS %8u/%c' len %6u %8.4f%% -- %8u/%c' len %6u VS %8u/%c' len %6u %8.4f%% -- ACCEPTED\n", fi, this5->fragId(), this5->frag3p() ? '3' : '5', this5ovlLen, that5->fragId(), that5->frag3p() ? '3' : '5', that5ovlLen, percDiff5, this3->fragId(), this3->frag3p() ? '3' : '5', this3ovlLen, that3->fragId(), that3->frag3p() ? '3' : '5', that3ovlLen, percDiff3); #endif nAccepted++; } else { #if 0 writeLog("fi %8u -- %8u/%c' len %6u VS %8u/%c' len %6u %8.4f%% -- %8u/%c' len %6u VS %8u/%c' len %6u %8.4f%%\n", fi, this5->fragId(), this5->frag3p() ? '3' : '5', this5ovlLen, that5->fragId(), that5->frag3p() ? '3' : '5', that5ovlLen, percDiff5, this3->fragId(), this3->frag3p() ? '3' : '5', this3ovlLen, that3->fragId(), that3->frag3p() ? '3' : '5', that3ovlLen, percDiff3); #endif nRejected++; #pragma omp critical (suspInsert) _suspicious.insert(fi); } } writeLog("BestOverlapGraph()-- suspicious %u contained %u spur %u mutual-best %u accepted %u rejected %u\n", nSuspicious, nContained, nSpur, nMutual, nAccepted, nRejected); } void BestOverlapGraph::removeSpurs(void) { uint32 fiLimit = FI->numFragments(); uint32 numThreads = omp_get_max_threads(); uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99; writeLog("BestOverlapGraph()-- detecting spur fragments.\n"); _spur.clear(); for (uint32 fi=1; fi <= fiLimit; fi++) { bool spur5 = (getBestEdgeOverlap(fi, false)->fragId() == 0); bool spur3 = (getBestEdgeOverlap(fi, true)->fragId() == 0); if (isContained(fi)) // Contained, not a spur. continue; if ((spur5 == false) && (spur3 == false)) // Edges off of both ends. Not a spur. continue; if ((spur5 == true) && (spur3 == true)) // No edges off either end. Not a spur, just garbage. continue; // Exactly one end is missing a best edge. Bad! writeLog("BestOverlapGraph()-- frag "F_U32" is a %s spur.\n", fi, (spur5) ? "5'" : "3'"); _spur.insert(fi); } } void BestOverlapGraph::findEdges(void) { uint32 fiLimit = FI->numFragments(); uint32 numThreads = omp_get_max_threads(); uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99; memset(_bestA, 0, sizeof(BestOverlaps) * (fiLimit + 1)); memset(_scorA, 0, sizeof(BestScores) * (fiLimit + 1)); writeLog("BestOverlapGraph()-- analyzing %d fragments for best contains, with %d threads.\n", fiLimit, numThreads); #pragma omp parallel for schedule(dynamic, blockSize) for (uint32 fi=1; fi <= fiLimit; fi++) { uint32 no = 0; BAToverlap *ovl = OC->getOverlaps(fi, AS_MAX_EVALUE, no); for (uint32 ii=0; iigetOverlaps(fi, AS_MAX_EVALUE, no); // Build edges out of spurs, but don't allow edges into them. This should prevent them from // being incorporated into a promiscuous unitig, but still let them be popped as bubbles (but // they shouldn't because they're spurs). for (uint32 ii=0; iinumFragments(); writeLog("BestOverlapGraph()-- removing best edges for contained fragments.\n"); for (uint32 fi=1; fi <= fiLimit; fi++) { if (isContained(fi) == true) { getBestEdgeOverlap(fi, false)->clear(); getBestEdgeOverlap(fi, true) ->clear(); } } } BestOverlapGraph::BestOverlapGraph(double erateGraph, double deviationGraph, const char *prefix) { setLogFile(prefix, "bestOverlapGraph"); writeLog("BestOverlapGraph-- allocating best edges ("F_SIZE_T"MB)\n", ((2 * sizeof(BestEdgeOverlap) * (FI->numFragments() + 1)) >> 20)); _bestA = new BestOverlaps [FI->numFragments() + 1]; // Cleared in findEdges() _scorA = new BestScores [FI->numFragments() + 1]; _mean = erateGraph; _stddev = 0.0; _median = erateGraph; _mad = 0.0; _suspicious.clear(); _bestM.clear(); _scorM.clear(); _restrict = NULL; _restrictEnabled = false; _erateGraph = erateGraph; _deviationGraph = deviationGraph; // Mark reads as suspicious if they are not fully covered by overlaps. removeSuspicious(); findEdges(); if (logFileFlagSet(LOG_ALL_BEST_EDGES)) reportBestEdges(prefix, "best.0.initial"); // Analyze the current best edges to set a cutoff on overlap quality used for graph building. removeHighErrorBestEdges(); findEdges(); if (logFileFlagSet(LOG_ALL_BEST_EDGES)) reportBestEdges(prefix, "best.1.filtered"); // Mark reads as suspicious if the length of the best edge out is very different than the length // of the best edge that should be back to us. E.g., if readA has best edge to readB (of length // lenAB), but readB has best edge to readC (of length lenBC), and lenAB is much shorter than // lenBC, then something is wrong with readA. // // This must come before removeSpurs(). removeLopsidedEdges(); findEdges(); if (logFileFlagSet(LOG_ALL_BEST_EDGES)) reportBestEdges(prefix, "best.2.cleaned"); // Mark reads as spurs, so we don't find best edges to them. removeSpurs(); findEdges(); reportBestEdges(prefix, logFileFlagSet(LOG_ALL_BEST_EDGES) ? "best.3.final" : "best"); // One more pass, to find any ambiguous best edges. // Cleanup the contained reads. Why? removeContainedDovetails(); // Done with scoring data. delete [] _scorA; _scorA = NULL; _spur.clear(); setLogFile(prefix, NULL); } BestOverlapGraph::BestOverlapGraph(double erateGraph, double deviationGraph, set *restrict) { _erateGraph = erateGraph; _deviationGraph = deviationGraph; _bestA = NULL; _scorA = NULL; _bestM.clear(); _scorM.clear(); assert(restrict != NULL); _restrict = restrict; _restrictEnabled = true; // PASS 0: Load the map (necessary?) #if 0 for (set::iterator it=_restrict->begin(); it != _restrict->end(); it++) { uint32 fi = *it; _bestM[fi].insert(); _scorM[fi].insert(); } #endif // PASS 1: Find containments. for (set::iterator it=_restrict->begin(); it != _restrict->end(); it++) { uint32 fi = *it; uint32 no = 0; BAToverlap *ovl = OC->getOverlaps(fi, AS_MAX_EVALUE, no); for (uint32 ii=0; ii::iterator it=_restrict->begin(); it != _restrict->end(); it++) { uint32 fi = *it; uint32 no = 0; BAToverlap *ovl = OC->getOverlaps(fi, AS_MAX_EVALUE, no); for (uint32 ii=0; ii::iterator it=_restrict->begin(); it != _restrict->end(); it++) { uint32 fi = *it; if (isContained(fi) == true) { getBestEdgeOverlap(fi, false)->clear(); getBestEdgeOverlap(fi, true) ->clear(); } } // Remove spurs #if 0 for (set::iterator it=_restrict->begin(); it != _restrict->end(); it++) { uint32 fi = *it; if ((getBestEdgeOverlap(fi, false)->fragId() == 0) || (getBestEdgeOverlap(fi, true)->fragId() == 0)) { getBestEdgeOverlap(fi, false)->clear(); getBestEdgeOverlap(fi, true) ->clear(); } } #endif _restrict = NULL; _restrictEnabled = false; } void BestOverlapGraph::reportBestEdges(const char *prefix, const char *label) { char N[FILENAME_MAX]; FILE *BCH = NULL; FILE *BE = NULL, *BEH = NULL, *BEG; FILE *BS = NULL; FILE *SS = NULL; sprintf(N, "%s.%s.edges", prefix, label); BE = fopen(N, "w"); sprintf(N, "%s.%s.singletons", prefix, label); BS = fopen(N, "w"); sprintf(N, "%s.%s.edges.suspicious", prefix, label); SS = fopen(N, "w"); sprintf(N, "%s.%s.contains.histogram", prefix, label); BCH = fopen(N, "w"); sprintf(N, "%s.%s.edges.histogram", prefix, label); BEH = fopen(N, "w"); sprintf(N, "%s.%s.edges.gfa", prefix, label); BEG = fopen(N, "w"); if ((BE) && (BS)) { fprintf(BE, "#fragId\tlibId\tbest5iid\tbest5end\tbest3iid\tbest3end\teRate5\teRate3\tbest5len\tbest3len\n"); fprintf(BS, "#fragId\tlibId\n"); for (uint32 id=1; idnumFragments() + 1; id++) { BestEdgeOverlap *bestedge5 = getBestEdgeOverlap(id, false); BestEdgeOverlap *bestedge3 = getBestEdgeOverlap(id, true); if ((bestedge5->fragId() == 0) && (bestedge3->fragId() == 0) && (isContained(id) == false)) { fprintf(BS, "%u\t%u\n", id, FI->libraryIID(id)); } else if (_suspicious.count(id) > 0) { fprintf(SS, "%u\t%u\t%u\t%c'\t%u\t%c'\t%6.4f\t%6.4f\t%u\t%u%s\n", id, FI->libraryIID(id), bestedge5->fragId(), bestedge5->frag3p() ? '3' : '5', bestedge3->fragId(), bestedge3->frag3p() ? '3' : '5', AS_OVS_decodeEvalue(bestedge5->evalue()), AS_OVS_decodeEvalue(bestedge3->evalue()), (bestedge5->fragId() == 0 ? 0 : FI->overlapLength(id, bestedge5->fragId(), bestedge5->ahang(), bestedge5->bhang())), (bestedge3->fragId() == 0 ? 0 : FI->overlapLength(id, bestedge3->fragId(), bestedge3->ahang(), bestedge3->bhang())), isContained(id) ? "\tcontained" : ""); } else { fprintf(BE, "%u\t%u\t%u\t%c'\t%u\t%c'\t%6.4f\t%6.4f\t%u\t%u%s\n", id, FI->libraryIID(id), bestedge5->fragId(), bestedge5->frag3p() ? '3' : '5', bestedge3->fragId(), bestedge3->frag3p() ? '3' : '5', AS_OVS_decodeEvalue(bestedge5->evalue()), AS_OVS_decodeEvalue(bestedge3->evalue()), (bestedge5->fragId() == 0 ? 0 : FI->overlapLength(id, bestedge5->fragId(), bestedge5->ahang(), bestedge5->bhang())), (bestedge3->fragId() == 0 ? 0 : FI->overlapLength(id, bestedge3->fragId(), bestedge3->ahang(), bestedge3->bhang())), isContained(id) ? "\tcontained" : ""); } } fclose(BE); fclose(BS); fclose(SS); } if (BEG) { fprintf(BEG, "H\tVN:Z:bogart/edges\n"); // First, write the sequences used. for (uint32 id=1; idnumFragments() + 1; id++) { BestEdgeOverlap *bestedge5 = getBestEdgeOverlap(id, false); BestEdgeOverlap *bestedge3 = getBestEdgeOverlap(id, true); if ((bestedge5->fragId() == 0) && (bestedge3->fragId() == 0) && (isContained(id) == false)) { // Do nothing, a singleton. } else if (isContained(id) == true) { // Do nothing, a contained read. } else if (_suspicious.count(id) > 0) { // Do nothing, a suspicious read. } else { // Report the read, it has best edges - including contained reads. fprintf(BEG, "S\tread%08u\t*\tLN:i:%u\n", id, FI->fragmentLength(id)); } } // Now, report edges. GFA wants edges in exactly this format: // // ------------- // ------------- // // with read orientation given by +/-. Conveniently, this is what we've saved (for the edges). for (uint32 id=1; idnumFragments() + 1; id++) { BestEdgeOverlap *bestedge5 = getBestEdgeOverlap(id, false); BestEdgeOverlap *bestedge3 = getBestEdgeOverlap(id, true); if ((bestedge5->fragId() == 0) && (bestedge3->fragId() == 0) && (isContained(id) == false)) { // Do nothing, a singleton. } else if (isContained(id) == true) { // Do nothing, a contained read. } else if (_suspicious.count(id) > 0) { // Do nothing, a suspicious read. } else { if (bestedge5->fragId() != 0) { int32 ahang = bestedge5->ahang(); int32 bhang = bestedge5->bhang(); int32 olaplen = FI->overlapLength(id, bestedge5->fragId(), bestedge5->ahang(), bestedge5->bhang()); assert((ahang <= 0) && (bhang <= 0)); // ALL 5' edges should be this. fprintf(BEG, "L\tread%08u\t-\tread%08u\t%c\t%uM\n", id, bestedge5->fragId(), bestedge5->frag3p() ? '-' : '+', olaplen); } if (bestedge3->fragId() != 0) { int32 ahang = bestedge3->ahang(); int32 bhang = bestedge3->bhang(); int32 olaplen = FI->overlapLength(id, bestedge3->fragId(), bestedge3->ahang(), bestedge3->bhang()); assert((ahang >= 0) && (bhang >= 0)); // ALL 3' edges should be this. fprintf(BEG, "L\tread%08u\t+\tread%08u\t%c\t%uM\n", id, bestedge3->fragId(), bestedge3->frag3p() ? '-' : '+', FI->overlapLength(id, bestedge3->fragId(), bestedge3->ahang(), bestedge3->bhang())); } } } fclose(BEG); } if ((BCH) && (BEH)) { double *bc = new double [FI->numFragments() + 1 + FI->numFragments() + 1]; double *be = new double [FI->numFragments() + 1 + FI->numFragments() + 1]; uint32 bcl = 0; uint32 bel = 0; for (uint32 id=1; idnumFragments() + 1; id++) { BestEdgeOverlap *bestedge5 = getBestEdgeOverlap(id, false); BestEdgeOverlap *bestedge3 = getBestEdgeOverlap(id, true); if (isContained(id)) { //bc[bcl++] = bestcont->erate(); #warning what is the error rate of the 'best contained' overlap? bc[bcl++] = bestedge5->erate(); bc[bcl++] = bestedge3->erate(); } else { if (bestedge5->fragId() > 0) be[bel++] = bestedge5->erate(); if (bestedge3->fragId() > 0) be[bel++] = bestedge3->erate(); } } sort(bc, bc+bcl); sort(be, be+bel); for (uint32 ii=0; ii olap.b_iid)) // Exact! Each contains the other. Make the lower IID the container. return; if ((olap.a_hang > 0) || (olap.b_hang < 0)) // We only save if A is the contained fragment. return; setContained(olap.a_iid); } void BestOverlapGraph::scoreEdge(const BAToverlap& olap) { bool enableLog = false; // useful for reporting this stuff only for specific reads //if ((olap.a_iid == 97202) || (olap.a_iid == 30701)) // enableLog = true; if (isOverlapBadQuality(olap)) { // Yuck. Don't want to use this crud. if ((enableLog == true) && (logFileFlagSet(LOG_OVERLAP_SCORING))) writeLog("scoreEdge()-- OVERLAP BADQ: %d %d %c hangs "F_S32" "F_S32" err %.3f -- bad quality\n", olap.a_iid, olap.b_iid, olap.flipped ? 'A' : 'N', olap.a_hang, olap.b_hang, olap.erate); return; } if (isOverlapRestricted(olap)) { // Whoops, don't want this overlap for this BOG if ((enableLog == true) && (logFileFlagSet(LOG_OVERLAP_SCORING))) writeLog("scoreEdge()-- OVERLAP RESTRICT: %d %d %c hangs "F_S32" "F_S32" err %.3f -- restricted\n", olap.a_iid, olap.b_iid, olap.flipped ? 'A' : 'N', olap.a_hang, olap.b_hang, olap.erate); return; } if (isSuspicious(olap.b_iid)) { // Whoops, don't want this overlap for this BOG if ((enableLog == true) && (logFileFlagSet(LOG_OVERLAP_SCORING))) writeLog("scoreEdge()-- OVERLAP SUSP: %d %d %c hangs "F_S32" "F_S32" err %.3f -- suspicious\n", olap.a_iid, olap.b_iid, olap.flipped ? 'A' : 'N', olap.a_hang, olap.b_hang, olap.erate); return; } if (((olap.a_hang >= 0) && (olap.b_hang <= 0)) || ((olap.a_hang <= 0) && (olap.b_hang >= 0))) { // Skip containment overlaps. if ((enableLog == true) && (logFileFlagSet(LOG_OVERLAP_SCORING))) writeLog("scoreEdge()-- OVERLAP CONT: %d %d %c hangs "F_S32" "F_S32" err %.3f -- container read\n", olap.a_iid, olap.b_iid, olap.flipped ? 'A' : 'N', olap.a_hang, olap.b_hang, olap.erate); return; } if (isContained(olap.b_iid) == true) { // Skip overlaps to contained reads (allow scoring of best edges from contained reads). if ((enableLog == true) && (logFileFlagSet(LOG_OVERLAP_SCORING))) writeLog("scoreEdge()-- OVERLAP CONT: %d %d %c hangs "F_S32" "F_S32" err %.3f -- contained read\n", olap.a_iid, olap.b_iid, olap.flipped ? 'A' : 'N', olap.a_hang, olap.b_hang, olap.erate); return; } uint64 newScr = scoreOverlap(olap); bool a3p = olap.AEndIs3prime(); BestEdgeOverlap *best = getBestEdgeOverlap(olap.a_iid, a3p); uint64 &score = (a3p) ? (best3score(olap.a_iid)) : (best5score(olap.a_iid)); assert(newScr > 0); if (newScr <= score) { if ((enableLog == true) && (logFileFlagSet(LOG_OVERLAP_SCORING))) writeLog("scoreEdge()-- OVERLAP GOOD: %d %d %c hangs "F_S32" "F_S32" err %.3f -- no better than best\n", olap.a_iid, olap.b_iid, olap.flipped ? 'A' : 'N', olap.a_hang, olap.b_hang, olap.erate); return; } best->set(olap); score = newScr; if ((enableLog == true) && (logFileFlagSet(LOG_OVERLAP_SCORING))) writeLog("scoreEdge()-- OVERLAP BEST: %d %d %c hangs "F_S32" "F_S32" err %.3f -- NOW BEST\n", olap.a_iid, olap.b_iid, olap.flipped ? 'A' : 'N', olap.a_hang, olap.b_hang, olap.erate); } bool BestOverlapGraph::isOverlapBadQuality(const BAToverlap& olap) { bool enableLog = false; // useful for reporting this stuff only for specific reads //if ((olap.a_iid == 97202) || (olap.a_iid == 30701)) // enableLog = true; if ((FI->fragmentLength(olap.a_iid) == 0) || (FI->fragmentLength(olap.b_iid) == 0)) // The overlap is bad if it involves deleted fragments. Shouldn't happen in a normal // assembly, but sometimes us users want to delete fragments after overlaps are generated. return(true); // The overlap is GOOD (false == not bad) if the error rate is below the allowed erate. // Initially, this is just the erate passed in. After the first rount of finding edges, // it is reset to the mean and stddev of selected best edges. // double Tstddev = _mean + _deviationGraph * _stddev; double Tmad = _median + _deviationGraph * 1.4826 * _mad; if (olap.erate <= Tmad || (Tmad == 0 && olap.erate <= Tstddev)) { if ((enableLog == true) && (logFileFlagSet(LOG_OVERLAP_SCORING))) writeLog("isOverlapBadQuality()-- OVERLAP GOOD: %d %d %c hangs "F_S32" "F_S32" err %.3f\n", olap.a_iid, olap.b_iid, olap.flipped ? 'A' : 'N', olap.a_hang, olap.b_hang, olap.erate); return(false); } // CA8.3 would further compute the (expected) number of errors in the alignment, and compare // against another limit. This was to allow very short overlaps where one error would push the // error rate above a few percent. canu doesn't do short overlaps. if ((enableLog == true) && (logFileFlagSet(LOG_OVERLAP_SCORING))) writeLog("isOverlapBadQuality()-- OVERLAP REJECTED: %d %d %c hangs "F_S32" "F_S32" err %.3f\n", olap.a_iid, olap.b_iid, olap.flipped ? 'A' : 'N', olap.a_hang, olap.b_hang, olap.erate); return(true); } // If no restrictions are known, this overlap is useful if both fragments are not in a unitig // already. Otherwise, we are restricted to just a specific set of fragments (usually a single // unitig and all the mated reads). The overlap is useful if both fragments are in the set. // bool BestOverlapGraph::isOverlapRestricted(const BAToverlap &olap) { if (_restrictEnabled == false) return(false); assert(_restrict != NULL); if ((_restrict->count(olap.a_iid) != 0) && (_restrict->count(olap.b_iid) != 0)) return(false); else return(true); } uint64 BestOverlapGraph::scoreOverlap(const BAToverlap& olap) { uint64 leng = 0; uint64 rate = AS_MAX_EVALUE - olap.evalue; assert(olap.evalue <= AS_MAX_EVALUE); assert(rate <= AS_MAX_EVALUE); // Containments - the length of the overlaps are all the same. We return the quality. // if (((olap.a_hang >= 0) && (olap.b_hang <= 0)) || ((olap.a_hang <= 0) && (olap.b_hang >= 0))) return(rate); // Dovetails - the length of the overlap is the score, but we bias towards lower error. // (again, shift AFTER assigning to avoid overflows) assert((olap.a_hang < 0) == (olap.b_hang < 0)); // Compute the length of the overlap, as either the official overlap length that currently // takes into account both reads, or as the number of aligned bases on the A read. #if 0 leng = FI->overlapLength(olap.a_iid, olap.b_iid, olap.a_hang, olap.b_hang); #endif if (olap.a_hang > 0) leng = FI->fragmentLength(olap.a_iid) - olap.a_hang; else leng = FI->fragmentLength(olap.a_iid) + olap.b_hang; // Convert the length into an expected number of matches. #if 0 assert(olap.erate <= 1.0); leng -= leng * olap.erate; #endif // And finally shift it to the correct place in the word. leng <<= AS_MAX_EVALUE_BITS; return(leng | rate); } canu-1.3/src/bogart/AS_BAT_BestOverlapGraph.H000066400000000000000000000170611272335422300207520ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_BestOverlapGraph.H * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2014-JAN-29 * are Copyright 2010-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-21 to 2015-JUN-03 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-MAR-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_BESTOVERLAPGRAPH #define INCLUDE_AS_BAT_BESTOVERLAPGRAPH #include "AS_global.H" #include "AS_BAT_OverlapCache.H" class FragmentEnd { public: FragmentEnd() { _id = 0; _e3p = false; }; FragmentEnd(uint32 id, bool e3p) { _id = id; _e3p = e3p; }; uint32 fragId(void) const { return(_id); }; bool frag3p(void) const { return(_e3p == true); }; bool frag5p(void) const { return(_e3p == false); }; bool operator==(FragmentEnd const that) const { return((fragId() == that.fragId()) && (frag3p() == that.frag3p())); }; bool operator!=(FragmentEnd const that) const { return((fragId() != that.fragId()) || (frag3p() != that.frag3p())); }; bool operator<(FragmentEnd const that) const { if (fragId() != that.fragId()) return fragId() < that.fragId(); else return frag3p() < that.frag3p(); }; private: uint32 _id:31; uint32 _e3p:1; }; // Stores an overlap from an 'a' read (implied by the index into the array of best edges) to a 'b' // read. The hangs are relative to the 'a' read - just as a normal overlap would be. // class BestEdgeOverlap { public: BestEdgeOverlap() { clear(); }; BestEdgeOverlap(BAToverlap const &ovl) { set(ovl); }; ~BestEdgeOverlap() { }; void clear(void) { _id = 0; _e3p = 0; _ahang = 0; _bhang = 0; _evalue = 0; }; void set(BAToverlap const &olap) { _id = olap.b_iid; if (((olap.a_hang <= 0) && (olap.b_hang >= 0)) || // If contained, _e3p just means ((olap.a_hang >= 0) && (olap.b_hang <= 0))) // the other read is flipped _e3p = olap.flipped; else _e3p = olap.BEndIs3prime(); // Otherwise, means olap is to the 3' end _ahang = olap.a_hang; _bhang = olap.b_hang; _evalue = olap.evalue; }; void set(uint32 id, bool e3p, int32 ahang, int32 bhang, uint32 evalue) { _id = id; _e3p = e3p; _ahang = ahang; _bhang = bhang; _evalue = evalue; }; uint32 fragId(void) const { return(_id); }; bool frag3p(void) const { return(_e3p == true); }; bool frag5p(void) const { return(_e3p == false); }; int32 ahang(void) const { return(_ahang); }; int32 bhang(void) const { return(_bhang); }; uint32 evalue(void) const { return(_evalue); }; double erate(void) const { return(AS_OVS_decodeEvalue(_evalue)); }; private: uint32 _id; uint64 _e3p : 1; // Overlap with the 3' end of that fragment, or flipped contain int64 _ahang : AS_MAX_READLEN_BITS+1; int64 _bhang : AS_MAX_READLEN_BITS+1; uint64 _evalue : AS_MAX_EVALUE_BITS; }; #if (1 + AS_MAX_READLEN_BITS + 1 + AS_MAX_READLEN_BITS + 1 + AS_MAX_EVALUE_BITS > 64) #error not enough bits to store overlaps. decrease AS_MAX_EVALUE_BITS or AS_MAX_READLEN_BITS. #endif class BestOverlaps { public: BestEdgeOverlap _best5; BestEdgeOverlap _best3; uint32 _isC; }; class BestScores { public: BestScores() { _best5score = 0; _best3score = 0; _isC = 0; }; uint64 _best5score; uint64 _best3score; uint32 _isC; }; class BestOverlapGraph { private: void removeSuspicious(void); void removeSpurs(void); void removeLopsidedEdges(void); void findEdges(void); void removeHighErrorBestEdges(void); void removeContainedDovetails(void); public: BestOverlapGraph(double erateGraph, double deviationGraph, const char *prefix); BestOverlapGraph(double erateGraph, double deviationGraph, set *restrict); ~BestOverlapGraph() { delete [] _bestA; delete [] _scorA; }; // Given a fragment UINT32 and which end, returns pointer to // BestOverlap node. BestEdgeOverlap *getBestEdgeOverlap(uint32 fragid, bool threePrime) { if (_bestA) return((threePrime) ? (&_bestA[fragid]._best3) : (&_bestA[fragid]._best5)); return((threePrime) ? (&_bestM[fragid]._best3) : (&_bestM[fragid]._best5)); }; // given a FragmentEnd sets it to the next FragmentEnd after following the // best edge FragmentEnd followOverlap(FragmentEnd end) { if (end.fragId() == 0) return(FragmentEnd()); BestEdgeOverlap *edge = getBestEdgeOverlap(end.fragId(), end.frag3p()); return(FragmentEnd(edge->fragId(), !edge->frag3p())); }; void setContained(const uint32 fragid) { if (_bestA) _bestA[fragid]._isC = true; else _bestM[fragid]._isC = true; }; bool isContained(const uint32 fragid) { if (_bestA) return(_bestA[fragid]._isC); return(_bestM[fragid]._isC); }; bool isSuspicious(const uint32 fragid) { return(_suspicious.count(fragid) > 0); }; void reportBestEdges(const char *prefix, const char *label); public: bool isOverlapBadQuality(const BAToverlap& olap); // Used in repeat detection private: bool isOverlapRestricted(const BAToverlap &olap); uint64 scoreOverlap(const BAToverlap& olap); private: void scoreContainment(const BAToverlap& olap); void scoreEdge(const BAToverlap& olap); private: uint64 &best5score(uint32 id) { if (_restrictEnabled == false) return(_scorA[id]._best5score); return(_scorM[id]._best5score); }; uint64 &best3score(uint32 id) { if (_restrictEnabled == false) return(_scorA[id]._best3score); return(_scorM[id]._best3score); }; private: BestOverlaps *_bestA; BestScores *_scorA; double _mean; double _stddev; double _median; double _mad; set _suspicious; set _spur; map _bestM; map _scorM; set *_restrict; bool _restrictEnabled; public: double _erateGraph; double _deviationGraph; }; //BestOverlapGraph extern BestOverlapGraph *OG; #endif // INCLUDE_AS_BAT_BESTOVERLAPGRAPH canu-1.3/src/bogart/AS_BAT_ChunkGraph.C000066400000000000000000000161051272335422300175650ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_ChunkGraph.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010,2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2014-DEC-22 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_ChunkGraph.H" #include "AS_BAT_Logging.H" ChunkGraph::ChunkGraph(const char *output_prefix) { setLogFile(output_prefix, "ChunkGraph"); _maxFragment = FI->numFragments(); _restrict = NULL; _pathLen = new uint32 [_maxFragment * 2 + 2]; _chunkLength = new ChunkLength [_maxFragment]; _chunkLengthIter = 0; memset(_pathLen, 0, sizeof(uint32) * (_maxFragment * 2 + 2)); memset(_chunkLength, 0, sizeof(ChunkLength) * (_maxFragment)); for (uint32 fid=1; fid <= _maxFragment; fid++) { if (OG->isContained(fid)) { if (logFileFlagSet(LOG_CHUNK_GRAPH)) writeLog("read %u contained\n", fid); continue; } if (OG->isSuspicious(fid)) { if (logFileFlagSet(LOG_CHUNK_GRAPH)) writeLog("read %u suspicious\n", fid); continue; } uint32 l5 = countFullWidth(FragmentEnd(fid, false)); uint32 l3 = countFullWidth(FragmentEnd(fid, true)); _chunkLength[fid-1].fragId = fid; _chunkLength[fid-1].cnt = l5 + l3; } delete [] _pathLen; _pathLen = NULL; std::sort(_chunkLength, _chunkLength + _maxFragment); } ChunkGraph::ChunkGraph(set *restrict) { _maxFragment = 0; _restrict = restrict; for (set::iterator it=_restrict->begin(); it != _restrict->end(); it++) _idMap[*it] = _maxFragment++; _pathLen = new uint32 [_maxFragment * 2 + 2]; _chunkLength = new ChunkLength [_maxFragment]; _chunkLengthIter = 0; memset(_pathLen, 0, sizeof(uint32) * (_maxFragment * 2 + 2)); memset(_chunkLength, 0, sizeof(ChunkLength) * (_maxFragment)); for (set::iterator it=_restrict->begin(); it != _restrict->end(); it++) { uint32 fid = *it; // Actual fragment ID uint32 fit = _idMap[fid]; // Local array index if (OG->isContained(fid)) continue; _chunkLength[fit].fragId = fid; _chunkLength[fit].cnt = (countFullWidth(FragmentEnd(fid, false)) + countFullWidth(FragmentEnd(fid, true))); } delete [] _pathLen; _pathLen = NULL; std::sort(_chunkLength, _chunkLength + _maxFragment); } uint64 ChunkGraph::getIndex(FragmentEnd e) { if (_restrict == NULL) return(e.fragId() * 2 + e.frag3p()); return(_idMap[e.fragId()] * 2 + e.frag3p()); } uint32 ChunkGraph::countFullWidth(FragmentEnd firstEnd) { uint64 firstIdx = getIndex(firstEnd); assert(firstIdx < _maxFragment * 2 + 2); if (_pathLen[firstIdx] > 0) return _pathLen[firstIdx]; uint32 length = 0; std::set seen; FragmentEnd lastEnd = firstEnd; uint64 lastIdx = firstIdx; // Until we run off the chain, or we hit a fragment with a known length, compute the length FROM // THE START. // while ((lastIdx != 0) && (_pathLen[lastIdx] == 0)) { seen.insert(lastEnd); _pathLen[lastIdx] = ++length; // Follow the path of lastEnd lastEnd = OG->followOverlap(lastEnd); lastIdx = getIndex(lastEnd); } // Check why we stopped. Three cases: // // 1) We ran out of best edges to follow -- lastEnd.fragId() == 0 // 2) We encountered a fragment with known length -- _pathLen[lastEnd.index()] > 0 // 3) We encountered a self-loop (same condition as case 2) // // To distinguish case 2 and 3, we keep a set<> of the fragments we've seen in this construction. // If 'lastEnd' is in that set, then we're case 3. If so, adjust every node in the cycle to have // the same length, the length of the cycle itself. // // 'lastEnd' and 'index' are the first fragment in the cycle; we've seen this one before. // if (lastEnd.fragId() == 0) { // Case 1. Do nothing. ; } else if (seen.find(lastEnd) != seen.end()) { // Case 3, a cycle. uint32 cycleLen = length - _pathLen[lastIdx] + 1; FragmentEnd currEnd = lastEnd; uint64 currIdx = lastIdx; do { _pathLen[currIdx] = cycleLen; currEnd = OG->followOverlap(currEnd); currIdx = getIndex(currEnd); } while (lastEnd != currEnd); } else { // Case 2, an existing path. length += _pathLen[lastIdx]; } // Our return value is now whatever count we're at. uint32 lengthMax = length; // Traverse again, converting "path length from the start" into "path length from the end". Any // cycle has had its length set correctly already, and we stop at either the start of the cycle, // or at the start of any existing path. // FragmentEnd currEnd = firstEnd; uint64 currIdx = firstIdx; while (currEnd != lastEnd) { _pathLen[currIdx] = length--; currEnd = OG->followOverlap(currEnd); currIdx = getIndex(currEnd); } if (logFileFlagSet(LOG_CHUNK_GRAPH)) { seen.clear(); currEnd = firstEnd; currIdx = firstIdx; writeLog("path from %d,%d length %d:", firstEnd.fragId(), (firstEnd.frag3p()) ? 3 : 5, _pathLen[firstIdx]); while ((currEnd.fragId() != 0) && (seen.find(currEnd) == seen.end())) { seen.insert(currEnd); if (currEnd == lastEnd) writeLog(" LAST"); writeLog(" %d,%d(%d)", currEnd.fragId(), (currEnd.frag3p()) ? 3 : 5, _pathLen[currIdx]); currEnd = OG->followOverlap(currEnd); currIdx = getIndex(currEnd); } if (seen.find(currEnd) != seen.end()) writeLog(" CYCLE %d,%d(%d)", currEnd.fragId(), (currEnd.frag3p()) ? 3 : 5, _pathLen[currIdx]); writeLog("\n"); } if (lengthMax != _pathLen[firstIdx]) writeLog("ERROR: lengthMax %d _pathLen[] %d\n", lengthMax, _pathLen[firstIdx]); assert(lengthMax == _pathLen[firstIdx]); return(_pathLen[firstIdx]); } canu-1.3/src/bogart/AS_BAT_ChunkGraph.H000066400000000000000000000050411272335422300175670ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_ChunkGraph.H * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010,2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2014-DEC-22 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_CHUNKGRAPH #define INCLUDE_AS_BAT_CHUNKGRAPH #include "AS_global.H" #include #include using namespace std; class BestOverlapGraph; class ChunkLength { public: uint32 fragId; uint32 cnt; bool operator<(ChunkLength const that) const { if (cnt == that.cnt) return(fragId < that.fragId); return(cnt > that.cnt); }; }; class ChunkGraph { public: ChunkGraph(const char *output_prefix); ChunkGraph(set *restrict); ~ChunkGraph(void) { delete [] _chunkLength; }; uint32 nextFragByChunkLength(void) { if (_chunkLengthIter >= _maxFragment) return(0); return(_chunkLength[_chunkLengthIter++].fragId); }; private: uint64 getIndex(FragmentEnd e); uint32 countFullWidth(FragmentEnd firstEnd); uint64 _maxFragment; // The usual case, for a chunk graph of all fragments. ChunkLength *_chunkLength; uint32 _chunkLengthIter; uint32 *_pathLen; // For a chunk graph of a single unitig plus some extra fragments. // This maps the uint32 to an index in the arrays above. map _idMap; set *_restrict; }; extern ChunkGraph *CG; #endif // INCLUDE_AS_BAT_CHUNKGRAPH canu-1.3/src/bogart/AS_BAT_FragmentInfo.C000066400000000000000000000131751272335422300201160ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_FragmentInfo.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2015-JUN-16 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_Logging.H" const uint64 fiMagicNumber = 0x6f666e4967617266llu; // 'fragInfo' until it gets messed up by endianess. const uint64 fiVersionNumber = 2; FragmentInfo::FragmentInfo(gkStore *gkp, const char *prefix, uint32 minReadLen) { if (load(prefix)) return; writeLog("FragmentInfo()-- Loading fragment information\n"); if (minReadLen > 0) writeLog("FragmentInfo()-- Reads shorter than "F_U32" bases are forced to be singleton.\n", minReadLen); _numLibraries = gkp->gkStore_getNumLibraries(); _numFragments = gkp->gkStore_getNumReads(); _fragLength = new uint32 [_numFragments + 1]; _libIID = new uint32 [_numFragments + 1]; for (uint32 i=0; i<_numFragments + 1; i++) { _fragLength[i] = 0; _libIID[i] = 0; } uint32 numSkipped = 0; uint32 numLoaded = 0; for (uint32 fi=1; fi<=_numFragments; fi++) { gkRead *read = gkp->gkStore_getRead(fi); if (read->gkRead_sequenceLength() < minReadLen) { numSkipped++; } else { uint32 iid = read->gkRead_readID(); uint32 lib = read->gkRead_libraryID(); _fragLength[iid] = read->gkRead_sequenceLength(); _libIID[iid] = lib; numLoaded++; } if (((numSkipped + numLoaded) % 10000000) == 0) writeLog("FragmentInfo()-- Loading fragment information: skipped:%9d active:%9d\n", numSkipped, numLoaded); } writeLog("FragmentInfo()-- Loaded %d alive reads, skipped %d short reads.\n", numLoaded, numSkipped); save(prefix); } FragmentInfo::~FragmentInfo() { delete [] _fragLength; delete [] _libIID; } void FragmentInfo::save(const char *prefix) { char name[FILENAME_MAX]; sprintf(name, "%s.fragmentInfo", prefix); errno = 0; FILE *file = fopen(name, "w"); if (errno) { writeLog("FragmentInfo()-- Failed to open '%s' for writing: %s\n", name, strerror(errno)); writeLog("FragmentInfo()-- Will not save fragment information to cache.\n"); return; } writeLog("FragmentInfo()-- Saving fragment information to cache '%s'\n", name); AS_UTL_safeWrite(file, &fiMagicNumber, "fragmentInformationMagicNumber", sizeof(uint64), 1); AS_UTL_safeWrite(file, &fiVersionNumber, "fragmentInformationMagicNumber", sizeof(uint64), 1); AS_UTL_safeWrite(file, &_numFragments, "fragmentInformationNumFrgs", sizeof(uint32), 1); AS_UTL_safeWrite(file, &_numLibraries, "fragmentInformationNumLibs", sizeof(uint32), 1); AS_UTL_safeWrite(file, _fragLength, "fragmentInformationFragLen", sizeof(uint32), _numFragments + 1); AS_UTL_safeWrite(file, _libIID, "fragmentInformationLibIID", sizeof(uint32), _numFragments + 1); fclose(file); } bool FragmentInfo::load(const char *prefix) { char name[FILENAME_MAX]; sprintf(name, "%s.fragmentInfo", prefix); errno = 0; FILE *file = fopen(name, "r"); if (errno) return(false); uint64 magicNumber = 0; uint64 versionNumber = 0; AS_UTL_safeRead(file, &magicNumber, "fragmentInformationMagicNumber", sizeof(uint64), 1); AS_UTL_safeRead(file, &versionNumber, "fragmentInformationVersionNumber", sizeof(uint64), 1); AS_UTL_safeRead(file, &_numFragments, "fragmentInformationNumFrgs", sizeof(uint32), 1); AS_UTL_safeRead(file, &_numLibraries, "fragmentInformationNumLibs", sizeof(uint32), 1); if (magicNumber != fiMagicNumber) { writeLog("FragmentInfo()-- File '%s' is not a fragment info; cannot load.\n", name); fclose(file); return(false); } if (versionNumber != fiVersionNumber) { writeLog("FragmentInfo()-- File '%s' is version "F_U64", I can only read version "F_U64"; cannot load.\n", name, versionNumber, fiVersionNumber); fclose(file); return(false); } writeLog("FragmentInfo()-- Loading fragment information for "F_U32" fragments and "F_U32" libraries from cache '%s'\n", _numFragments, _numLibraries, name); _fragLength = new uint32 [_numFragments + 1]; _libIID = new uint32 [_numFragments + 1]; AS_UTL_safeRead(file, _fragLength, "fragmentInformationFragLen", sizeof(uint32), _numFragments + 1); AS_UTL_safeRead(file, _libIID, "fragmentInformationLibIID", sizeof(uint32), _numFragments + 1); fclose(file); return(true); } canu-1.3/src/bogart/AS_BAT_FragmentInfo.H000066400000000000000000000070641272335422300201230ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/bogart/AS_BAT_Datatypes.H * * Modifications by: * * Brian P. Walenz beginning on 2016-APR-28 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_FRAGMENT_INFO #define INCLUDE_AS_BAT_FRAGMENT_INFO #include "AS_global.H" #include "ovStore.H" #include "gkStore.H" #include #include #include #include #include class FragmentInfo { public: FragmentInfo(gkStore *gkp, const char *prefix, uint32 minReadLen); ~FragmentInfo(); uint64 memoryUsage(void) { return((3 * sizeof(uint32) * _numFragments) + (2 * sizeof(double) * _numLibraries) + (2 * sizeof(uint32) * _numLibraries)); }; uint32 numFragments(void) { return(_numFragments); }; uint32 numLibraries(void) { return(_numLibraries); }; uint32 fragmentLength(uint32 iid) { return(_fragLength[iid]); }; uint32 libraryIID(uint32 iid) { return(_libIID[iid]); }; uint32 overlapLength(uint32 a_iid, uint32 b_iid, int32 a_hang, int32 b_hang) { int32 alen = fragmentLength(a_iid); int32 blen = fragmentLength(b_iid); int32 aovl = 0; int32 bovl = 0; assert(alen > 0); assert(blen > 0); if (a_hang < 0) { // b_hang < 0 ? ---------- : ---- // ? ---------- : ---------- // aovl = (b_hang < 0) ? (alen + b_hang) : (alen); bovl = (b_hang < 0) ? (blen + a_hang) : (blen + a_hang - b_hang); } else { // b_hang < 0 ? ---------- : ---------- // ? ---- : ---------- // aovl = (b_hang < 0) ? (alen - a_hang + b_hang) : (alen - a_hang); bovl = (b_hang < 0) ? (blen) : (blen - b_hang); } if ((aovl <= 0) || (bovl <= 0) || (aovl > alen) || (bovl > blen)) { fprintf(stderr, "WARNING: bogus overlap found for A="F_U32" B="F_U32"\n", a_iid, b_iid); fprintf(stderr, "WARNING: A len="F_S32" hang="F_S32" ovl="F_S32"\n", alen, a_hang, aovl); fprintf(stderr, "WARNING: B len="F_S32" hang="F_S32" ovl="F_S32"\n", blen, b_hang, bovl); } if (aovl < 0) aovl = 0; if (bovl < 0) bovl = 0; if (aovl > alen) aovl = alen; if (bovl > blen) bovl = blen; assert(aovl > 0); assert(bovl > 0); assert(aovl <= alen); assert(bovl <= blen); // AVE does not work. return((uint32)((aovl, bovl)/2)); // MAX does not work. return((uint32)MAX(aovl, bovl)); return(aovl); }; private: void save(const char *prefix); bool load(const char *prefix); uint32 _numFragments; uint32 _numLibraries; uint32 *_fragLength; uint32 *_libIID; }; extern FragmentInfo *FI; #endif // INCLUDE_AS_BAT_FRAGMENT_INFO canu-1.3/src/bogart/AS_BAT_Instrumentation.C000066400000000000000000001057001272335422300207360ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_Instrumentation.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-27 * are Copyright 2010-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2014-DEC-23 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_SetParentAndHang.H" #include "AS_BAT_Outputs.H" #include "intervalList.H" // Will fail if a read is in unitig 0, or if a read isn't in a unitig. void checkUnitigMembership(UnitigVector &unitigs) { uint32 *inUnitig = new uint32 [FI->numFragments()+1]; uint32 noUnitig = 0xffffffff; // All reads start of not placed in a unitig. for (uint32 i=0; inumFragments()+1; i++) inUnitig[i] = noUnitig; // Over all unitigs, remember where each read is. for (uint32 ti=0; tiufpath.size(); fi++) { ufNode *frg = &tig->ufpath[fi]; if (frg->ident > FI->numFragments()) fprintf(stderr, "tig %u ufpath[%d] ident %u more than number of reads %u\n", tig->id(), fi, frg->ident, FI->numFragments()); if (inUnitig[frg->ident] != noUnitig) fprintf(stderr, "tig %u ufpath[%d] ident %u placed multiple times\n", tig->id(), fi, frg->ident); assert(frg->ident <= FI->numFragments()); // Can't be out of range. assert(inUnitig[frg->ident] == noUnitig); // Read must be not placed yet. inUnitig[frg->ident] = ti; } } // Find any read not placed in a unitig. for (uint32 i=0; inumFragments()+1; i++) { if (FI->fragmentLength(i) == 0) // Deleted read. continue; assert(inUnitig[i] != 0); // There shouldn't be a unitig 0. assert(inUnitig[i] != noUnitig); // The read should be in a unitig. } delete [] inUnitig; } // Decides if a unitig is unassembled. The other classifications (isBubble, isCircular, isRepeat) // are made when the type is processed (e.g., when bubbles are popped). // // A unitig is unassembled if: // 1) it has fewer than R reads (R=2) // 2) it is shorter than S bases (S=1000) // 3) a single read spans at least fraction F of the lenth (F=1.0) // 4) at least fraction F of the unitig is below read depth D (F=1.0, D=2) // void classifyUnitigsAsUnassembled(UnitigVector &unitigs, uint32 fewReadsNumber, uint32 tooShortLength, double spanFraction, double lowcovFraction, uint32 lowcovDepth) { uint32 nTooFew = 0; uint32 nShort = 0; uint32 nSingle = 0; uint32 nCoverage = 0; uint32 nContig = 0; uint64 bTooFew = 0; uint64 bShort = 0; uint64 bSingle = 0; uint64 bCoverage = 0; uint64 bContig = 0; writeLog("==> FILTERING UNASSEMBLED CRUD\n"); for (uint32 ti=0; ti_isUnassembled = false; // Rule 1. Too few reads. if (utg->ufpath.size() < fewReadsNumber) { writeLog("unitig %u unassembled - too few reads (%u < %u)\n", ti, utg->ufpath.size(), fewReadsNumber); utg->_isUnassembled = true; nTooFew += 1; bTooFew += utg->getLength(); continue; } // Rule 2. Short. if (utg->getLength() < tooShortLength) { writeLog("unitig %u unassembled - too short (%u < %u)\n", ti, utg->getLength(), tooShortLength); utg->_isUnassembled = true; nShort += 1; bShort += utg->getLength(); continue; } // Rule 3. Single read spans large fraction of tig. for (uint32 oi=0; oiufpath.size(); oi++) { ufNode *frg = &utg->ufpath[oi]; int frgbgn = MIN(frg->position.bgn, frg->position.end); int frgend = MAX(frg->position.bgn, frg->position.end); if (frgend - frgbgn > utg->getLength() * spanFraction) { writeLog("unitig %u unassembled - single read spans unitig (read %u %u-%u spans fraction %f > %f\n", ti, frg->ident, frg->position.bgn, frg->position.end, (double)(frgend - frgbgn) / utg->getLength(), spanFraction); utg->_isUnassembled = true; nSingle += 1; bSingle += utg->getLength(); break; } } if (utg->_isUnassembled) continue; // Rule 4. Low coverage. intervalList IL; for (uint32 oi=0; oiufpath.size(); oi++) { ufNode *frg = &utg->ufpath[oi]; int frgbgn = MIN(frg->position.bgn, frg->position.end); int frgend = MAX(frg->position.bgn, frg->position.end); IL.add(frgbgn, frgend - frgbgn); } intervalList ID(IL); uint32 basesLow = 0; uint32 basesHigh = 0; for (uint32 ii=0; ii= lowcovFraction) { writeLog("Unitig %u unassembled - low coverage (%.4f > %.4f at < %ux coverage)\n", ti, lowcov, lowcovFraction, lowcovDepth); utg->_isUnassembled = true; nCoverage += 1; bCoverage += utg->getLength(); continue; } // Otherwise, unitig is assembled! nContig += 1; bContig += utg->getLength(); } writeLog("unassembled filter: %6u unitigs %11lu bases -- too few reads\n", nTooFew, bTooFew); writeLog("unassembled filter: %6u unitigs %11lu bases -- too short\n", nShort, bShort); writeLog("unassembled filter: %6u unitigs %11lu bases -- single spanning read\n", nSingle, bSingle); writeLog("unassembled filter: %6u unitigs %11lu bases -- low coverage\n", nCoverage, bCoverage); writeLog("unassembled filter: %6u unitigs %11lu bases -- acceptable contigs\n", nContig, bContig); } void reportN50(vector &data, char const *label, uint64 genomeSize) { uint64 cnt = data.size(); uint64 sum = 0; uint64 tot = 0; uint64 nnn = 10; uint64 siz = 0; if (cnt == 0) return; // Duplicates tgTigSizeAnalysis::printSummary() sort(data.begin(), data.end(), greater()); for (uint64 i=0; i 0) siz = genomeSize; else siz = tot; for (uint64 i=0; i unassembledLength; vector bubbleLength; vector repeatLength; vector circularLength; vector contigLength; for (uint32 ti=0; ti_isUnassembled) { unassembledLength.push_back(utg->getLength()); } else if (utg->_isBubble) { bubbleLength.push_back(utg->getLength()); } else if (utg->_isRepeat) { repeatLength.push_back(utg->getLength()); } else if (utg->_isCircular) { circularLength.push_back(utg->getLength()); } else { contigLength.push_back(utg->getLength()); } } reportN50(unassembledLength, "UNASSEMBLED", genomeSize); reportN50(bubbleLength, "BUBBLE", genomeSize); reportN50(repeatLength, "REPEAT", genomeSize); reportN50(circularLength, "CIRCULAR", genomeSize); reportN50(contigLength, "CONTIGS", genomeSize); if (logFileFlagSet(LOG_INTERMEDIATE_UNITIGS) == 0) return; // Dump to an intermediate store. char tigStorePath[FILENAME_MAX]; sprintf(tigStorePath, "%s.%03u.%s.tigStore", prefix, logFileOrder, name); fprintf(stderr, "Creating intermediate tigStore '%s'\n", tigStorePath); uint32 numFragsT = 0; uint32 numFragsP = 0; uint64 utgLen = 0; // Compute average frags per partition. for (uint32 ti=0; tiufpath.size(); if (utg->ufpath.size() > 2) utgLen += utg->getLength(); } if (utgLen < 16 * 1024 * 1024) numFragsP = numFragsT / 7; else if (utgLen < 64 * 1024 * 1024) numFragsP = numFragsT / 63; else numFragsP = numFragsT / 127; // Dump the unitigs to an intermediate store. setParentAndHang(unitigs); writeUnitigsToStore(unitigs, tigStorePath, tigStorePath, numFragsP, false); } #define tCTG 0 // To a read in a normal tig #define tRPT 1 // To a read in a repeat tig #define tBUB 2 // To a read in a bubble tig #define tUNA 3 // To a read in an 'unassembled' leftover tig #define tUNU 4 // To a read not placed in a tig #define tNOP 5 // To no read (for best edges) struct olapsUsed { uint64 total; // By definition, satisfied overlaps are in the same tig. uint64 doveSatSame[6]; uint64 contSatSame[6]; // Unsatisfied overlaps can be in the same tig... uint64 doveUnsatSame[6]; uint64 contUnsatSame[6]; // ...or can be between tigs. uint64 doveUnsatDiff[6][6]; uint64 contUnsatDiff[6][6]; }; uint32 getTigType(Unitig *tg) { if (tg == NULL) return(tUNU); if (tg->_isUnassembled) return(tUNA); if (tg->_isBubble) return(tBUB); if (tg->_isRepeat) return(tRPT); if (1) return(tCTG); } bool satisfiedOverlap(uint32 rdAlo, uint32 rdAhi, bool rdAfwd, uint32 rdBlo, uint32 rdBhi, bool rdBfwd, bool flipped) { return(((rdAhi < rdBlo) || (rdBhi < rdBlo)) || // Not satisfied, no overlap ((rdAfwd == rdBfwd) && (flipped == true)) || // Not satisfied, same orient, but flipped overlap ((rdAfwd != rdBfwd) && (flipped == false))); // Not satisfied, diff orient, but normal overlap } // Iterate over all overlaps (but the only interface we have is by iterating // over all reads), and count the number of overlaps satisfied in unitigs. void reportOverlaps(UnitigVector &unitigs, const char *prefix, const char *name) { olapsUsed *dd = new olapsUsed; // Dovetail overlaps to non-contained reads olapsUsed *dc = new olapsUsed; // Dovetail overlaps to contained reads olapsUsed *cc = new olapsUsed; // Containment overlaps olapsUsed *bb = new olapsUsed; // Best overlaps memset(dd, 0, sizeof(olapsUsed)); memset(dc, 0, sizeof(olapsUsed)); memset(cc, 0, sizeof(olapsUsed)); memset(bb, 0, sizeof(olapsUsed)); for (uint32 fi=0; finumFragments()+1; fi++) { if (FI->fragmentLength(fi) == 0) continue; uint32 rdAid = fi; uint32 tgAid = Unitig::fragIn(rdAid); Unitig *tgA = unitigs[tgAid]; uint32 tgAtype = getTigType(tgA); // Best overlaps exist if the read isn't contained. if (OG->isContained(rdAid) == false) { BestEdgeOverlap *b5 = OG->getBestEdgeOverlap(fi, false); uint32 rd5id = b5->fragId(); uint32 tg5id = Unitig::fragIn(rd5id); Unitig *tg5 = unitigs[tg5id]; uint32 tg5type = getTigType(tg5); BestEdgeOverlap *b3 = OG->getBestEdgeOverlap(fi, true); uint32 rd3id = b3->fragId(); uint32 tg3id = Unitig::fragIn(rd3id); Unitig *tg3 = unitigs[tg3id]; uint32 tg3type = getTigType(tg3); bb->total += 2; // If this read isn't even in a tig, add to the unused categories. if (tgAid == 0) { if (rd5id == 0) bb->doveUnsatDiff[tUNU][tNOP]++; else bb->doveUnsatDiff[tUNU][tg5type]++; if (rd3id == 0) bb->doveUnsatDiff[tUNU][tNOP]++; else bb->doveUnsatDiff[tUNU][tg3type]++; } // Otherwise, its in a tig, and we need to compare positions. else { uint32 rdApos = unitigs[tgAid]->pathPosition(rdAid); ufNode *rdA = &unitigs[tgAid]->ufpath[rdApos]; bool rdAfwd = (rdA->position.bgn < rdA->position.end); int32 rdAlo = (rdAfwd) ? rdA->position.bgn : rdA->position.end; int32 rdAhi = (rdAfwd) ? rdA->position.end : rdA->position.bgn; // Different tigs? Unsatisfied. Same tig? Grab the reads and check for overlap. if (tgA != tg5) { bb->doveUnsatDiff[tgAtype][tg5type]++; } else if (rd5id == 0) { bb->doveUnsatDiff[tgAtype][tNOP]++; } else { uint32 rd5pos = unitigs[tg5id]->pathPosition(rd5id); ufNode *rd5 = &unitigs[tg5id]->ufpath[rd5pos]; bool rd5fwd = (rd5->position.bgn < rd5->position.end); int32 rd5lo = (rd5fwd) ? rd5->position.bgn : rd5->position.end; int32 rd5hi = (rd5fwd) ? rd5->position.end : rd5->position.bgn; if (satisfiedOverlap(rdAlo, rdAhi, rdAfwd, rd5lo, rd5hi, rd5fwd, (b5->frag3p() == true))) { bb->doveSatSame[tgAtype]++; } else { bb->doveUnsatSame[tgAtype]++; } } if (tgA != tg3) { bb->doveUnsatDiff[tgAtype][tg3type]++; } else if (rd3id == 0) { bb->doveUnsatDiff[tgAtype][tNOP]++; } else { uint32 rd3pos = unitigs[tg3id]->pathPosition(rd3id); ufNode *rd3 = &unitigs[tg3id]->ufpath[rd3pos]; bool rd3fwd = (rd3->position.bgn < rd3->position.end); int32 rd3lo = (rd3fwd) ? rd3->position.bgn : rd3->position.end; int32 rd3hi = (rd3fwd) ? rd3->position.end : rd3->position.bgn; if (satisfiedOverlap(rdAlo, rdAhi, rdAfwd, rd3lo, rd3hi, rd3fwd, (b3->frag3p() == false))) { bb->doveSatSame[tgAtype]++; } else { bb->doveUnsatSame[tgAtype]++; } } } } // For all overlaps. uint32 ovlLen = 0; BAToverlap *ovl = OC->getOverlaps(fi, AS_MAX_ERATE, ovlLen); for (uint32 oi=0; oiisContained(rdAid) || OG->isContained(rdBid); // Figure out what class of overlap we're counting. olapsUsed *used = NULL; if (isDove == false) used = cc; else if (contReads == true) used = dc; else used = dd; used->total++; // If to reads not in a tig, unsatisfied. if ((tgAid == 0) || (tgBid == 0)) { if (isDove) used->doveUnsatDiff[tgAtype][tgBtype]++; else used->contUnsatDiff[tgAtype][tgBtype]++; continue; } // If in different tigs, unsatisfied. if (tgAid != tgBid) { if (isDove) used->doveUnsatDiff[tgAtype][tgBtype]++; else used->contUnsatDiff[tgAtype][tgBtype]++; continue; } // Else, possibly satisfied. We need to check positions. uint32 rdApos = unitigs[tgAid]->pathPosition(rdAid); ufNode *rdA = &unitigs[tgAid]->ufpath[rdApos]; bool rdAfwd = (rdA->position.bgn < rdA->position.end); int32 rdAlo = (rdAfwd) ? rdA->position.bgn : rdA->position.end; int32 rdAhi = (rdAfwd) ? rdA->position.end : rdA->position.bgn; uint32 rdBpos = unitigs[tgBid]->pathPosition(rdBid); ufNode *rdB = &unitigs[tgBid]->ufpath[rdBpos]; bool rdBfwd = (rdB->position.bgn < rdB->position.end); int32 rdBlo = (rdBfwd) ? rdB->position.bgn : rdB->position.end; int32 rdBhi = (rdBfwd) ? rdB->position.end : rdB->position.bgn; // If overlapping and correctly oriented, good enough for now. Do we want to care about // overlap length? Nah, there's enough fudging (still, I think) in placement that it'd be // tough to get that usefully precise. if (satisfiedOverlap(rdAlo, rdAhi, rdAfwd, rdBlo, rdBhi, rdBfwd, ovl[oi].flipped)) { if (isDove) used->doveUnsatSame[tgAtype]++; else used->contUnsatSame[tgAtype]++; } else { if (isDove) used->doveSatSame[tgAtype]++; else used->contSatSame[tgAtype]++; } } } // Merge the symmetrical counts for (uint32 ii=0; ii<6; ii++) { for (uint32 jj=ii+1; jj<6; jj++) { bb->doveUnsatDiff[ii][jj] += bb->doveUnsatDiff[jj][ii]; bb->doveUnsatDiff[jj][ii] = UINT64_MAX; dd->doveUnsatDiff[ii][jj] += dd->doveUnsatDiff[jj][ii]; dd->doveUnsatDiff[jj][ii] = UINT64_MAX; dc->doveUnsatDiff[ii][jj] += dc->doveUnsatDiff[jj][ii]; dc->doveUnsatDiff[jj][ii] = UINT64_MAX; cc->doveUnsatDiff[ii][jj] += cc->doveUnsatDiff[jj][ii]; cc->doveUnsatDiff[jj][ii] = UINT64_MAX; bb->contUnsatDiff[ii][jj] += bb->contUnsatDiff[jj][ii]; bb->contUnsatDiff[jj][ii] = UINT64_MAX; dd->contUnsatDiff[ii][jj] += dd->contUnsatDiff[jj][ii]; dd->contUnsatDiff[jj][ii] = UINT64_MAX; dc->contUnsatDiff[ii][jj] += dc->contUnsatDiff[jj][ii]; dc->contUnsatDiff[jj][ii] = UINT64_MAX; cc->contUnsatDiff[ii][jj] += cc->contUnsatDiff[jj][ii]; cc->contUnsatDiff[jj][ii] = UINT64_MAX; } } // Emit a nicely formatted report. #define B(X) (100.0 * (X) / (bb->total)) #define P(X) (100.0 * (X) / (dd->total)) #define Q(X) (100.0 * (X) / (dc->total)) #define R(X) (100.0 * (X) / (cc->total)) writeLog("--------------------------------------------------------------------------------\n"); writeLog("OVERLAP FATE\n"); writeLog("\n"); writeLog("dovetail overlaps (best) "F_U64"\n", bb->total); writeLog("dovetail overlaps "F_U64"\n", dd->total); writeLog("dovetail overlaps to contained reads "F_U64"\n", dc->total); writeLog("containment overlaps "F_U64"\n", cc->total); writeLog("\n"); writeLog("SATISFIED best edges DOVETAIL\n"); writeLog("--------- ------------ -------\n"); writeLog("contig %12"F_U64P" %6.2f%%\n", bb->doveSatSame[tCTG], B(bb->doveSatSame[tCTG])); writeLog("repeat contig %12"F_U64P" %6.2f%%\n", bb->doveSatSame[tRPT], B(bb->doveSatSame[tRPT])); writeLog("bubble %12"F_U64P" %6.2f%%\n", bb->doveSatSame[tBUB], B(bb->doveSatSame[tBUB])); writeLog("\n"); writeLog("UNSATISFIED best edges DOVETAIL\n"); writeLog("----------- ------------ -------\n"); writeLog("contig %12"F_U64P" %6.2f%%\n", bb->doveUnsatSame[tCTG], B(bb->doveUnsatSame[tCTG])); writeLog("repeat %12"F_U64P" %6.2f%%\n", bb->doveUnsatSame[tRPT], B(bb->doveUnsatSame[tRPT])); writeLog("bubble %12"F_U64P" %6.2f%%\n", bb->doveUnsatSame[tBUB], B(bb->doveUnsatSame[tBUB])); writeLog("unassembled %12"F_U64P" %6.2f%%\n", bb->doveUnsatSame[tUNA], B(bb->doveUnsatSame[tUNA])); writeLog("unused %12"F_U64P" %6.2f%%\n", bb->doveUnsatSame[tUNU], B(bb->doveUnsatSame[tUNU])); writeLog("\n"); writeLog("UNSATISFIED best edges DOVETAIL\n"); writeLog("----------- ------------ -------\n"); writeLog("contig-contig %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tCTG][tCTG], B(bb->doveUnsatDiff[tCTG][tCTG])); writeLog("contig-repeat %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tCTG][tRPT], B(bb->doveUnsatDiff[tCTG][tRPT])); writeLog("contig-bubble %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tCTG][tBUB], B(bb->doveUnsatDiff[tCTG][tBUB])); writeLog("contig-unassembled %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tCTG][tUNA], B(bb->doveUnsatDiff[tCTG][tUNA])); writeLog("contig-unused %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tCTG][tUNU], B(bb->doveUnsatDiff[tCTG][tUNU])); writeLog("contig-none %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tCTG][tNOP], B(bb->doveUnsatDiff[tCTG][tNOP])); writeLog("\n"); //writeLog("repeat-contig %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tRPT][tCTG], B(bb->doveUnsatDiff[tRPT][tCTG])); writeLog("repeat-repeat %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tRPT][tRPT], B(bb->doveUnsatDiff[tRPT][tRPT])); writeLog("repeat-bubble %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tRPT][tBUB], B(bb->doveUnsatDiff[tRPT][tBUB])); writeLog("repeat-unassembled %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tRPT][tUNA], B(bb->doveUnsatDiff[tRPT][tUNA])); writeLog("repeat-unused %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tRPT][tUNU], B(bb->doveUnsatDiff[tRPT][tUNU])); writeLog("repeat-none %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tRPT][tNOP], B(bb->doveUnsatDiff[tRPT][tNOP])); writeLog("\n"); //writeLog("bubble-contig %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tBUB][tCTG], B(bb->doveUnsatDiff[tBUB][tCTG])); //writeLog("bubble-repeat %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tBUB][tRPT], B(bb->doveUnsatDiff[tBUB][tRPT])); writeLog("bubble-bubble %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tBUB][tBUB], B(bb->doveUnsatDiff[tBUB][tBUB])); writeLog("bubble-unassembled %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tBUB][tUNA], B(bb->doveUnsatDiff[tBUB][tUNA])); writeLog("bubble-unused %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tBUB][tUNU], B(bb->doveUnsatDiff[tBUB][tUNU])); writeLog("bubble-none %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tBUB][tNOP], B(bb->doveUnsatDiff[tBUB][tNOP])); writeLog("\n"); //writeLog("unassembled-contig %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNA][tCTG], B(bb->doveUnsatDiff[tUNA][tCTG])); //writeLog("unassembled-repeat %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNA][tRPT], B(bb->doveUnsatDiff[tUNA][tRPT])); //writeLog("unassembled-bubble %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNA][tBUB], B(bb->doveUnsatDiff[tUNA][tBUB])); writeLog("unassembled-unassembled %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNA][tUNA], B(bb->doveUnsatDiff[tUNA][tUNA])); writeLog("unassembled-unused %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNA][tUNU], B(bb->doveUnsatDiff[tUNA][tUNU])); writeLog("unassembled-none %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNA][tNOP], B(bb->doveUnsatDiff[tUNA][tNOP])); writeLog("\n"); //writeLog("unused-contig %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNU][tCTG], B(bb->doveUnsatDiff[tUNU][tCTG])) //writeLog("unused-repeat %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNU][tRPT], B(bb->doveUnsatDiff[tUNU][tRPT])); //writeLog("unused-bubble %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNU][tBUB], B(bb->doveUnsatDiff[tUNU][tBUB])); //writeLog("unused-unassembled %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNU][tUNA], B(bb->doveUnsatDiff[tUNU][tUNA])); writeLog("unused-unused %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNU][tUNU], B(bb->doveUnsatDiff[tUNU][tUNU])); writeLog("unused-none %12"F_U64P" %6.2f%%\n", bb->doveUnsatDiff[tUNU][tNOP], B(bb->doveUnsatDiff[tUNU][tNOP])); writeLog("\n"); writeLog("\n"); writeLog("\n"); writeLog("SATISFIED all overlaps DOVETAIL DOVECONT CONTAINMENT\n"); writeLog("--------- ------------ ------- ------------ ------- ------------ -------\n"); writeLog("contig %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveSatSame[tCTG], P(dd->doveSatSame[tCTG]), dc->doveSatSame[tCTG], Q(dc->doveSatSame[tCTG]), cc->contSatSame[tCTG], R(cc->contSatSame[tCTG])); writeLog("repeat contig %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveSatSame[tRPT], P(dd->doveSatSame[tRPT]), dc->doveSatSame[tRPT], Q(dc->doveSatSame[tRPT]), cc->contSatSame[tRPT], R(cc->contSatSame[tRPT])); writeLog("bubble %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveSatSame[tBUB], P(dd->doveSatSame[tBUB]), dc->doveSatSame[tBUB], Q(dc->doveSatSame[tBUB]), cc->contSatSame[tBUB], R(cc->contSatSame[tBUB])); writeLog("\n"); writeLog("UNSATISFIED all overlaps DOVETAIL DOVECONT CONTAINMENT\n"); writeLog("----------- ------------ ------- ------------ ------- ------------ -------\n"); writeLog("contig %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatSame[tCTG], P(dd->doveUnsatSame[tCTG]), dc->doveUnsatSame[tCTG], Q(dc->doveUnsatSame[tCTG]), cc->contUnsatSame[tCTG], R(cc->contUnsatSame[tCTG])); writeLog("repeat %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatSame[tRPT], P(dd->doveUnsatSame[tRPT]), dc->doveUnsatSame[tRPT], Q(dc->doveUnsatSame[tRPT]), cc->contUnsatSame[tRPT], R(cc->contUnsatSame[tRPT])); writeLog("bubble %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatSame[tBUB], P(dd->doveUnsatSame[tBUB]), dc->doveUnsatSame[tBUB], Q(dc->doveUnsatSame[tBUB]), cc->contUnsatSame[tBUB], R(cc->contUnsatSame[tBUB])); writeLog("unassembled %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatSame[tUNA], P(dd->doveUnsatSame[tUNA]), dc->doveUnsatSame[tUNA], Q(dc->doveUnsatSame[tUNA]), cc->contUnsatSame[tUNA], R(cc->contUnsatSame[tUNA])); writeLog("unused %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatSame[tUNU], P(dd->doveUnsatSame[tUNU]), dc->doveUnsatSame[tUNU], Q(dc->doveUnsatSame[tUNU]), cc->contUnsatSame[tUNU], R(cc->contUnsatSame[tUNU])); writeLog("\n"); writeLog("UNSATISFIED all overlaps DOVETAIL DOVECONT CONTAINMENT\n"); writeLog("----------- ------------ ------- ------------ ------- ------------ -------\n"); writeLog("contig-contig %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tCTG][tCTG], P(dd->doveUnsatDiff[tCTG][tCTG]), dc->doveUnsatDiff[tCTG][tCTG], Q(dc->doveUnsatDiff[tCTG][tCTG]), cc->contUnsatDiff[tCTG][tCTG], R(cc->contUnsatDiff[tCTG][tCTG])); writeLog("contig-repeat %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tCTG][tRPT], P(dd->doveUnsatDiff[tCTG][tRPT]), dc->doveUnsatDiff[tCTG][tRPT], Q(dc->doveUnsatDiff[tCTG][tRPT]), cc->contUnsatDiff[tCTG][tRPT], R(cc->contUnsatDiff[tCTG][tRPT])); writeLog("contig-bubble %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tCTG][tBUB], P(dd->doveUnsatDiff[tCTG][tBUB]), dc->doveUnsatDiff[tCTG][tBUB], Q(dc->doveUnsatDiff[tCTG][tBUB]), cc->contUnsatDiff[tCTG][tBUB], R(cc->contUnsatDiff[tCTG][tBUB])); writeLog("contig-unassembled %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tCTG][tUNA], P(dd->doveUnsatDiff[tCTG][tUNA]), dc->doveUnsatDiff[tCTG][tUNA], Q(dc->doveUnsatDiff[tCTG][tUNA]), cc->contUnsatDiff[tCTG][tUNA], R(cc->contUnsatDiff[tCTG][tUNA])); writeLog("contig-unused %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tCTG][tUNU], P(dd->doveUnsatDiff[tCTG][tUNU]), dc->doveUnsatDiff[tCTG][tUNU], Q(dc->doveUnsatDiff[tCTG][tUNU]), cc->contUnsatDiff[tCTG][tUNU], R(cc->contUnsatDiff[tCTG][tUNU])); writeLog("\n"); //writeLog("repeat-contig %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tRPT][tCTG], P(dd->doveUnsatDiff[tRPT][tCTG]), dc->doveUnsatDiff[tRPT][tCTG], Q(dc->doveUnsatDiff[tRPT][tCTG]), cc->contUnsatDiff[tRPT][tCTG], R(cc->contUnsatDiff[tRPT][tCTG])); writeLog("repeat-repeat %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tRPT][tRPT], P(dd->doveUnsatDiff[tRPT][tRPT]), dc->doveUnsatDiff[tRPT][tRPT], Q(dc->doveUnsatDiff[tRPT][tRPT]), cc->contUnsatDiff[tRPT][tRPT], R(cc->contUnsatDiff[tRPT][tRPT])); writeLog("repeat-bubble %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tRPT][tBUB], P(dd->doveUnsatDiff[tRPT][tBUB]), dc->doveUnsatDiff[tRPT][tBUB], Q(dc->doveUnsatDiff[tRPT][tBUB]), cc->contUnsatDiff[tRPT][tBUB], R(cc->contUnsatDiff[tRPT][tBUB])); writeLog("repeat-unassembled %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tRPT][tUNA], P(dd->doveUnsatDiff[tRPT][tUNA]), dc->doveUnsatDiff[tRPT][tUNA], Q(dc->doveUnsatDiff[tRPT][tUNA]), cc->contUnsatDiff[tRPT][tUNA], R(cc->contUnsatDiff[tRPT][tUNA])); writeLog("repeat-unused %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tRPT][tUNU], P(dd->doveUnsatDiff[tRPT][tUNU]), dc->doveUnsatDiff[tRPT][tUNU], Q(dc->doveUnsatDiff[tRPT][tUNU]), cc->contUnsatDiff[tRPT][tUNU], R(cc->contUnsatDiff[tRPT][tUNU])); writeLog("\n"); //writeLog("bubble-contig %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tBUB][tCTG], P(dd->doveUnsatDiff[tBUB][tCTG]), dc->doveUnsatDiff[tBUB][tCTG], Q(dc->doveUnsatDiff[tBUB][tCTG]), cc->contUnsatDiff[tBUB][tCTG], R(cc->contUnsatDiff[tBUB][tCTG])); //writeLog("bubble-repeat %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tBUB][tRPT], P(dd->doveUnsatDiff[tBUB][tRPT]), dc->doveUnsatDiff[tBUB][tRPT], Q(dc->doveUnsatDiff[tBUB][tRPT]), cc->contUnsatDiff[tBUB][tRPT], R(cc->contUnsatDiff[tBUB][tRPT])); writeLog("bubble-bubble %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tBUB][tBUB], P(dd->doveUnsatDiff[tBUB][tBUB]), dc->doveUnsatDiff[tBUB][tBUB], Q(dc->doveUnsatDiff[tBUB][tBUB]), cc->contUnsatDiff[tBUB][tBUB], R(cc->contUnsatDiff[tBUB][tBUB])); writeLog("bubble-unassembled %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tBUB][tUNA], P(dd->doveUnsatDiff[tBUB][tUNA]), dc->doveUnsatDiff[tBUB][tUNA], Q(dc->doveUnsatDiff[tBUB][tUNA]), cc->contUnsatDiff[tBUB][tUNA], R(cc->contUnsatDiff[tBUB][tUNA])); writeLog("bubble-unused %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tBUB][tUNU], P(dd->doveUnsatDiff[tBUB][tUNU]), dc->doveUnsatDiff[tBUB][tUNU], Q(dc->doveUnsatDiff[tBUB][tUNU]), cc->contUnsatDiff[tBUB][tUNU], R(cc->contUnsatDiff[tBUB][tUNU])); writeLog("\n"); //writeLog("unassembled-contig %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tUNA][tCTG], P(dd->doveUnsatDiff[tUNA][tCTG]), dc->doveUnsatDiff[tUNA][tCTG], Q(dc->doveUnsatDiff[tUNA][tCTG]), cc->contUnsatDiff[tUNA][tCTG], R(cc->contUnsatDiff[tUNA][tCTG])); //writeLog("unassembled-repeat %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tUNA][tRPT], P(dd->doveUnsatDiff[tUNA][tRPT]), dc->doveUnsatDiff[tUNA][tRPT], Q(dc->doveUnsatDiff[tUNA][tRPT]), cc->contUnsatDiff[tUNA][tRPT], R(cc->contUnsatDiff[tUNA][tRPT])); //writeLog("unassembled-bubble %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tUNA][tBUB], P(dd->doveUnsatDiff[tUNA][tBUB]), dc->doveUnsatDiff[tUNA][tBUB], Q(dc->doveUnsatDiff[tUNA][tBUB]), cc->contUnsatDiff[tUNA][tBUB], R(cc->contUnsatDiff[tUNA][tBUB])); writeLog("unassembled-unassembled %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tUNA][tUNA], P(dd->doveUnsatDiff[tUNA][tUNA]), dc->doveUnsatDiff[tUNA][tUNA], Q(dc->doveUnsatDiff[tUNA][tUNA]), cc->contUnsatDiff[tUNA][tUNA], R(cc->contUnsatDiff[tUNA][tUNA])); writeLog("unassembled-unused %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tUNA][tUNU], P(dd->doveUnsatDiff[tUNA][tUNU]), dc->doveUnsatDiff[tUNA][tUNU], Q(dc->doveUnsatDiff[tUNA][tUNU]), cc->contUnsatDiff[tUNA][tUNU], R(cc->contUnsatDiff[tUNA][tUNU])); writeLog("\n"); //writeLog("unused-contig %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tUNU][tCTG], P(dd->doveUnsatDiff[tUNU][tCTG]), dc->doveUnsatDiff[tUNU][tCTG], Q(dc->doveUnsatDiff[tUNU][tCTG]), cc->contUnsatDiff[tUNU][tCTG], R(cc->contUnsatDiff[tUNU][tCTG])); //writeLog("unused-repeat %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tUNU][tRPT], P(dd->doveUnsatDiff[tUNU][tRPT]), dc->doveUnsatDiff[tUNU][tRPT], Q(dc->doveUnsatDiff[tUNU][tRPT]), cc->contUnsatDiff[tUNU][tRPT], R(cc->contUnsatDiff[tUNU][tRPT])); //writeLog("unused-bubble %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tUNU][tBUB], P(dd->doveUnsatDiff[tUNU][tBUB]), dc->doveUnsatDiff[tUNU][tBUB], Q(dc->doveUnsatDiff[tUNU][tBUB]), cc->contUnsatDiff[tUNU][tBUB], R(cc->contUnsatDiff[tUNU][tBUB])); //writeLog("unused-unassembled %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tUNU][tUNA], P(dd->doveUnsatDiff[tUNU][tUNA]), dc->doveUnsatDiff[tUNU][tUNA], Q(dc->doveUnsatDiff[tUNU][tUNA]), cc->contUnsatDiff[tUNU][tUNA], R(cc->contUnsatDiff[tUNU][tUNA])); writeLog("unused-unused %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%% %12"F_U64P" %6.2f%%\n", dd->doveUnsatDiff[tUNU][tUNU], P(dd->doveUnsatDiff[tUNU][tUNU]), dc->doveUnsatDiff[tUNU][tUNU], Q(dc->doveUnsatDiff[tUNU][tUNU]), cc->contUnsatDiff[tUNU][tUNU], R(cc->contUnsatDiff[tUNU][tUNU])); writeLog("\n"); writeLog("\n"); delete dd; delete dc; delete cc; } canu-1.3/src/bogart/AS_BAT_Instrumentation.H000066400000000000000000000037401272335422300207440ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_Instrumentation.H * * Modifications by: * * Brian P. Walenz from 2010-DEC-06 to 2013-AUG-01 * are Copyright 2010,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-19 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_INSTRUMENTATION #define INCLUDE_AS_BAT_INSTRUMENTATION void checkUnitigMembership(UnitigVector &unitigs); void reportOverlaps(UnitigVector &unitigs, const char *prefix, const char *name); void reportUnitigs(UnitigVector &unitigs, const char *prefix, const char *name, uint64 genomeSize); void classifyUnitigsAsUnassembled(UnitigVector &unitigs, uint32 fewReadsNumber, uint32 tooShortLength, double spanFraction, double lowcovFraction, uint32 lowcovDepth); #endif // INCLUDE_AS_BAT_INSTRUMENTATION canu-1.3/src/bogart/AS_BAT_Logging.C000066400000000000000000000146771272335422300171350ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_Logging.C * * Modifications by: * * Brian P. Walenz from 2012-JUL-29 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-19 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_Logging.H" class logFileInstance { public: logFileInstance() { file = stderr; name[0] = 0; part = 0; length = 0; }; ~logFileInstance() { if ((name[0] != 0) && (file)) { fprintf(stderr, "WARNING: open file '%s'\n", name); fclose(file); } }; void set(char const *prefix, int32 order, char const *label, int32 tn) { if (label == NULL) { file = stderr; name[0] = 0; part = 0; length = 0; return; } sprintf(name, "%s.%03u.%s.thr%03d", prefix, order, label, tn); }; void rotate(void) { assert(name[0] != 0); fclose(file); file = NULL; length = 0; part++; } void open(void) { char path[FILENAME_MAX]; assert(file == NULL); assert(name[0] != 0); sprintf(path, "%s.num%03d.log", name, part); errno = 0; file = fopen(path, "w"); if (errno) { fprintf(stderr, "setLogFile()-- Failed to open logFile '%s': %s.\n", path, strerror(errno)); fprintf(stderr, "setLogFile()-- Will now log to stderr instead.\n"); file = stderr; } }; void close(void) { if ((file != NULL) && (file != stderr)) fclose(file); file = NULL; name[0] = 0; part = 0; length = 0; }; FILE *file; char name[FILENAME_MAX]; uint32 part; uint64 length; }; // NONE of the logFileMain/logFileThread is implemented logFileInstance logFileMain; // For writes during non-threaded portions logFileInstance *logFileThread = NULL; // For writes during threaded portions. uint32 logFileOrder = 0; uint64 logFileFlags = 0; uint64 LOG_OVERLAP_SCORING = 0x0000000000000001; // Debug, scoring of overlaps uint64 LOG_ALL_BEST_EDGES = 0x0000000000000002; uint64 LOG_CHUNK_GRAPH = 0x0000000000000004; // Report the chunk graph as we build it uint64 LOG_BUILD_UNITIG = 0x0000000000000008; // Report building of initial unitigs (both unitig creation and fragment placement) uint64 LOG_PLACE_UNPLACED = 0x0000000000000010; // Report placing of unplaced reads uint64 LOG_BUBBLE_DETAIL = 0x0000000000000020; uint64 LOG_SPLIT_DISCONTINUOUS = 0x0000000000000040; // uint64 LOG_INTERMEDIATE_UNITIGS = 0x0000000000000080; // At various spots, dump the current unitigs uint64 LOG_SET_PARENT_AND_HANG = 0x0000000000000100; // uint64 LOG_STDERR = 0x0000000000000200; // Write ALL logging to stderr, not the files. uint64 LOG_PLACE_FRAG = 0x8000000000000000; // Internal use only. char const *logFileFlagNames[64] = { "overlapScoring", "allBestEdges", "chunkGraph", "buildUnitig", "placeUnplaced", "bubbles", "splitDiscontinuous", // Update made it to here, need repeats "intermediateUnitigs", "setParentAndHang", "stderr", NULL }; // Closes the current logFile, opens a new one called 'prefix.logFileOrder.label'. If 'label' is // NULL, the logFile is reset to stderr. void setLogFile(char const *prefix, char const *label) { assert(prefix != NULL); // Allocate space. if (logFileThread == NULL) logFileThread = new logFileInstance [omp_get_max_threads()]; // If writing to stderr, that's all we needed to do. if (logFileFlagSet(LOG_STDERR)) return; // Close out the old. logFileMain.close(); for (int32 tn=0; tnname[0] != 0) && (lf->length > maxLength)) { fprintf(lf->file, "logFile()-- size "F_U64" exceeds limit of "F_U64"; rotate to new file.\n", lf->length, maxLength); lf->rotate(); } // Open the file if needed. if (lf->file == NULL) lf->open(); // Write the log. va_start(ap, fmt); lf->length += vfprintf(lf->file, fmt, ap); va_end(ap); } void flushLog(void) { int32 nt = omp_get_num_threads(); int32 tn = omp_get_thread_num(); logFileInstance *lf = (nt == 1) ? (&logFileMain) : (&logFileThread[tn]); if (lf->file != NULL) fflush(lf->file); } canu-1.3/src/bogart/AS_BAT_Logging.H000066400000000000000000000045041272335422300171260ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_Logging.H * * Modifications by: * * Brian P. Walenz from 2012-JUL-29 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-AUG-11 to 2014-DEC-19 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_LOGGING #define INCLUDE_AS_BAT_LOGGING #include "AS_global.H" #include "AS_UTL_fileIO.H" #include #include #include #include #include #include #include #include #ifndef BROKEN_CLANG_OpenMP #include #endif void setLogFile(char const *prefix, char const *name); void writeLog(char const *fmt, ...); void flushLog(void); #define logFileFlagSet(L) ((logFileFlags & L) == L) extern uint64 logFileFlags; extern uint32 logFileOrder; // Used debug tigStore dumps, etc extern uint64 LOG_OVERLAP_SCORING; extern uint64 LOG_ALL_BEST_EDGES; extern uint64 LOG_CHUNK_GRAPH; extern uint64 LOG_BUILD_UNITIG; extern uint64 LOG_PLACE_UNPLACED; extern uint64 LOG_BUBBLE_DETAIL; extern uint64 LOG_SPLIT_DISCONTINUOUS; extern uint64 LOG_INTERMEDIATE_UNITIGS; extern uint64 LOG_SET_PARENT_AND_HANG; extern uint64 LOG_STDERR; extern uint64 LOG_PLACE_FRAG; extern char const *logFileFlagNames[64]; #endif // INCLUDE_AS_BAT_LOGGING canu-1.3/src/bogart/AS_BAT_MarkRepeatReads.C000066400000000000000000001147011272335422300205460ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAR-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_OverlapCache.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "intervalList.H" #include "stddev.H" #include using namespace std; // Hack. uint32 MIN_ANCHOR_HANG = 500; // Require reads to be anchored by this many bases at boundaries of repeats. int32 REPEAT_OVERLAP_MIN = 50; #define REPEAT_FRACTION 0.5 #undef SHOW_ANNOTATION_RAW // Show all overlaps used to annotate reads #undef SHOW_ANNOTATION_RAW_FILTERED // Show all overlaps filtered by high error rate #undef DUMP_READ_COVERAGE // Each evidence read picks its single best overlap to tig (based on overlaps to reads in the tig). // Filter out evidence that aligns at erate higher than expected. // Collapse to intervals on tig. // If still not significant and not spanned, break. class olapDat { public: olapDat(uint32 b, uint32 e, uint32 t, uint32 r) { tigbgn = b; tigend = e; eviTid = t; eviRid = r; }; bool operator<(const olapDat &that) const { return(tigbgn < that.tigbgn); }; uint32 tigbgn; // Location of the overlap on this tig uint32 tigend; // uint32 eviTid; // tig that the evidence read came from uint32 eviRid; // evidence read }; bool olapDatByEviRid(const olapDat &A, const olapDat &B) { if (A.eviRid == B.eviRid) return(A.tigbgn < B.tigbgn); return(A.eviRid < B.eviRid); } class breakPointCoords { public: breakPointCoords(uint32 tigID, int32 bgn, int32 end, bool rpt=false) { _tigID = tigID; _bgn = bgn; _end = end; _isRepeat = rpt; }; ~breakPointCoords() { }; bool operator<(breakPointCoords const &that) const { return(_bgn < that._bgn); }; uint32 _tigID; int32 _bgn; int32 _end; bool _isRepeat; }; // Returns the coordinates the overlap intersects on the A read. // // lo hi // v v // ------------ // ---------- void olapToReadCoords(ufNode *frg, int32 ahang, int32 bhang, int32 &lo, int32 &hi) { lo = 0; hi = FI->fragmentLength(frg->ident); if (ahang > 0) lo += ahang; // Positive hang! if (bhang < 0) hi += bhang; // Negative hang! assert(0 <= lo); assert(0 <= hi); assert(lo <= hi); assert(lo <= FI->fragmentLength(frg->ident)); assert(hi <= FI->fragmentLength(frg->ident)); } void findUnitigCoverage(Unitig *tig, intervalList &coverage) { intervalList rawcoverage; for (uint32 fi=0; fiufpath.size(); fi++) { ufNode frg = tig->ufpath[fi]; if (frg.position.bgn < frg.position.end) rawcoverage.add(frg.position.bgn, frg.position.end - frg.position.bgn); else rawcoverage.add(frg.position.end, frg.position.bgn - frg.position.end); } coverage.clear(); coverage.depth(rawcoverage); #ifdef DUMP_READ_COVERAGE char fn[FILENAME_MAX]; sprintf(fn, "%08u.coverage", tig->id()); FILE *F = fopen(fn, "w"); for (uint32 ii=0; ii &BP, Unitig **newTigs, int32 *lowCoord, uint32 *nRepeat, uint32 *nUnique, bool doMove) { uint32 nTigsCreated = 0; if (doMove == true) { memset(newTigs, 0, sizeof(Unitig *) * BP.size()); memset(lowCoord, 0, sizeof(int32) * BP.size()); } else { memset(nRepeat, 0, sizeof(uint32) * BP.size()); memset(nUnique, 0, sizeof(uint32) * BP.size()); } for (uint32 fi=0; fiufpath.size(); fi++) { ufNode &frg = tig->ufpath[fi]; int32 frgbgn = min(frg.position.bgn, frg.position.end); int32 frgend = max(frg.position.bgn, frg.position.end); // Search for the region that matches the read. BP's are sorted in increasing order. It // probably doesn't matter, but makes the logging a little easier to read. uint32 rid = UINT32_MAX; bool rpt = false; //fprintf(stderr, "Searching for placement for read %u at %u-%u\n", frg.ident, frgbgn, frgend); for (uint32 ii=0; ii nUnique[rid]) newTigs[rid]->_isRepeat = true; } newTigs[rid]->addFrag(frg, -lowCoord[rid], false); //LOG_ADDFRAG_BREAKING); } // Else, we're not moving, just count how many reads came from repeats or uniques. else { if (rpt) nRepeat[rid]++; else nUnique[rid]++; } } // Return the number of tigs created. for (uint32 ii=0; ii 0) nTigsCreated++; return(nTigsCreated); } // For each overlap, if the b-read is in this tig, ignore it. // Otherwise annotate the read with the overlap region. // // Later, check if the two reads in this unitig overlap; if not, annotate also. // void annotateRepeatsOnRead(UnitigVector &unitigs, Unitig *tgA, ufNode *rdA, double deviationRepeat, vector &repeats) { uint32 ovlLen = 0; BAToverlap *ovl = OC->getOverlaps(rdA->ident, AS_MAX_ERATE, ovlLen); vector readOlaps; // List of valid repeat overlaps to this read uint32 tgAid = tgA->id(); bool rdAfwd = (rdA->position.bgn < rdA->position.end); int32 rdAlo = (rdAfwd) ? rdA->position.bgn : rdA->position.end; int32 rdAhi = (rdAfwd) ? rdA->position.end : rdA->position.bgn; assert(rdAlo < rdAhi); // Beacuse the read is placed with a lot of fudging in the positions, we need // to scale the coordinates we compute here. double sc = (rdAhi - rdAlo) / (double)FI->fragmentLength(rdA->ident); // For all overlaps to this read, save the overlap if it is not represented in this tig. uint32 nOlaps = ovlLen; uint32 nDiff = 0; // Overlap to different tig uint32 nSelf = 0; // Overlap to same tig, different location uint32 nConf = 0; // Overlap to same tig, confirmed good overlap for (uint32 oi=0; oiufpath.size() == 1)) continue; // If the read is in an annotated bubble, skip. if (unitigs[tgBid]->_isBubble) continue; // If the overlap is to a container read, skip it. if ((ovl[oi].a_hang < 0) && (ovl[oi].b_hang > 0)) continue; // If the overlap is to a contained read, skip it. if ((ovl[oi].a_hang > 0) && (ovl[oi].b_hang < 0)) continue; uint32 rdBpos = unitigs[tgBid]->pathPosition(rdBid); ufNode *rdB = &unitigs[tgBid]->ufpath[rdBpos]; bool rdBfwd = (rdB->position.bgn < rdB->position.end); int32 rdBlo = (rdBfwd) ? rdB->position.bgn : rdB->position.end; int32 rdBhi = (rdBfwd) ? rdB->position.end : rdB->position.bgn; assert(rdBlo < rdBhi); // If the overlap is to a read in a different tig, save it. if (tgBid != tgAid) { nDiff++; olapToReadCoords(rdA, ovl[oi].a_hang, ovl[oi].b_hang, bgn, end); } // If the overlap is to a read in the same tig, but we don't overlap in the tig, save it. else if ((rdAhi < rdBlo) || (rdBhi < rdAlo)) { nSelf++; olapToReadCoords(rdA, ovl[oi].a_hang, ovl[oi].b_hang, bgn, end); } // Otherwise, the overlap is present in the tig, and can't indicate a repeat. else { nConf++; continue; } // Find the positions of the read that are covered by the overlap. int32 tigbgn = (rdAfwd) ? (rdAlo + sc * bgn) : (rdAhi - sc * end); int32 tigend = (rdAfwd) ? (rdAlo + sc * end) : (rdAhi - sc * bgn); assert(tigbgn < tigend); if (tigbgn < 0) tigbgn = 0; if (tigend > tgA->getLength()) tigend = tgA->getLength(); // Filter overlaps that are higher error than expected. double consistent = tgA->overlapConsistentWithTig(deviationRepeat, tigbgn, tigend, ovl[oi].erate); if (consistent < REPEAT_FRACTION) { #ifdef SHOW_ANNOTATION_RAW_FILTERED writeLog("tig %6u read %7u %8u-%8u OVERLAP from tig %6u read %7u %8u-%8u at tigpos %8u-%8u erate %.6f consistent %.4f FILTERED\n", tgAid, rdA->ident, rdAlo, rdAhi, tgBid, rdBid, rdBlo, rdBhi, tigbgn, tigend, ovl[oi].erate, consistent); #endif continue; } #ifdef SHOW_ANNOTATION_RAW writeLog("tig %6u read %7u %8u-%8u OVERLAP from tig %6u read %7u %8u-%8u at tigpos %8u-%8u erate %.6f consistent %.4f\n", tgAid, rdA->ident, rdAlo, rdAhi, tgBid, rdBid, rdBlo, rdBhi, tigbgn, tigend, ovl[oi].erate, consistent); #endif readOlaps.push_back(olapDat(tigbgn, tigend, tgBid, rdBid)); } // All overlaps processed. Save to the master list. #pragma omp critical (repeatsPushBack) for (uint32 rr=0; rr repeatOlaps; // Overlaps to reads promoted to tig coords intervalList tigMarksR; // Marked repeats based on reads, filtered by spanning reads intervalList tigMarksU; // Non-repeat invervals, just the inversion of tigMarksR for (uint32 ti=0; tiufpath.size() == 1) continue; vector repeats; writeLog("Annotating repeats in reads for tig %u/%u.\n", ti, tiLimit); // Clear out all the existing marks. They're not for this tig. // Analyze overlaps for each read. For each overlap to a read not in this tig, or not // overlapping in this tig, and of acceptable error rate, add the overlap to repeatOlaps. repeatOlaps.clear(); uint32 fiLimit = tig->ufpath.size(); uint32 numThreads = omp_get_max_threads(); uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99; #pragma omp parallel for if(fiLimit > 100) schedule(dynamic, blockSize) for (uint32 fi=0; fiufpath[fi], deviationRepeat, repeatOlaps); writeLog("Annotated with %lu overlaps.\n", repeatOlaps.size()); // Merge marks for the same read into the largest possible. sort(repeatOlaps.begin(), repeatOlaps.end(), olapDatByEviRid); #ifdef SHOW_ANNOTATE for (uint32 ii=0; iiufpath.size(); fi++) { ufNode *frg = &tig->ufpath[fi]; bool frgfwd = (frg->position.bgn < frg->position.end); int32 frglo = (frgfwd) ? frg->position.bgn : frg->position.end; int32 frghi = (frgfwd) ? frg->position.end : frg->position.bgn; bool discarded = false; for (uint32 ri=0; rigetLength()) && // Read at end of tig, spans off the low end (frglo + MIN_ANCHOR_HANG <= tigMarksR.lo(ri))) spanLo = spanHi = true; if (frglo + MIN_ANCHOR_HANG <= tigMarksR.lo(ri)) // Read spanned off the low end spanLo = true; if (tigMarksR.hi(ri) + MIN_ANCHOR_HANG <= frghi) // Read spanned off the high end spanHi = true; if (spanLo && spanHi) { writeLog("discard region %8d:%-8d - contained in read %6u %8d-%8d\n", tigMarksR.lo(ri), tigMarksR.hi(ri), frg->ident, frglo, frghi); tigMarksR.lo(ri) = 0; tigMarksR.hi(ri) = 0; discarded = true; } } if (discarded) tigMarksR.filterShort(1); } // Run through again, looking for the thickest overlap(s) to the remaining regions. // This isn't caring about the end effect noted above. #if 1 writeLog("thickest edges to the repeat regions:\n"); for (uint32 ri=0; riufpath.size(); fi++) { ufNode *frg = &tig->ufpath[fi]; bool frgfwd = (frg->position.bgn < frg->position.end); int32 frglo = (frgfwd) ? frg->position.bgn : frg->position.end; int32 frghi = (frgfwd) ? frg->position.end : frg->position.bgn; bool discarded = false; // Overlap off the 5' end of the region. if (frglo <= tigMarksR.lo(ri) && (tigMarksR.lo(ri) <= frghi)) { uint32 olap = frghi - tigMarksR.lo(ri); if (l5 < olap) { l5 = olap; t5 = fi; t5bgn = frglo; // Easier than recomputing it later on... t5end = frghi; } } // Overlap off the 3' end of the region. if (frglo <= tigMarksR.hi(ri) && (tigMarksR.hi(ri) <= frghi)) { uint32 olap = tigMarksR.hi(ri) - frglo; if (l3 < olap) { l3 = olap; t3 = fi; t3bgn = frglo; t3end = frghi; } } if (frglo <= tigMarksR.lo(ri) && (tigMarksR.hi(ri) <= frghi)) { writeLog("saved region %8d:%-8d - closest read %6u (%+6d) %8d:%-8d (%+6d) (contained)\n", tigMarksR.lo(ri), tigMarksR.hi(ri), frg->ident, tigMarksR.lo(ri) - frglo, frglo, frghi, frghi - tigMarksR.hi(ri)); } } if (t5 != UINT32_MAX) writeLog("saved region %8d:%-8d - closest 5' read %6u (%+6d) %8d:%-8d (%+6d)\n", tigMarksR.lo(ri), tigMarksR.hi(ri), tig->ufpath[t5].ident, tigMarksR.lo(ri) - t5bgn, t5bgn, t5end, t5end - tigMarksR.hi(ri)); if (t3 != UINT32_MAX) writeLog("saved region %8d:%-8d - closest 3' read %6u (%+6d) %8d:%-8d (%+6d)\n", tigMarksR.lo(ri), tigMarksR.hi(ri), tig->ufpath[t3].ident, tigMarksR.lo(ri) - t3bgn, t3bgn, t3end, t3end - tigMarksR.hi(ri)); } #endif // Scan reads. If a read intersects a repeat interval, and the best edge for that read // is entirely in the repeat region, decide if there is a near-best edge to something // not in this tig. // // A region with no such near-best edges is _probably_ correct. writeLog("search for confused edges:\n"); uint32 *isConfused = new uint32 [tigMarksR.numberOfIntervals()]; memset(isConfused, 0, sizeof(uint32) * tigMarksR.numberOfIntervals()); for (uint32 fi=0; fiufpath.size(); fi++) { ufNode *rdA = &tig->ufpath[fi]; uint32 rdAid = rdA->ident; bool rdAfwd = (rdA->position.bgn < rdA->position.end); int32 rdAlo = (rdAfwd) ? rdA->position.bgn : rdA->position.end; int32 rdAhi = (rdAfwd) ? rdA->position.end : rdA->position.bgn; double sc = (rdAhi - rdAlo) / (double)FI->fragmentLength(rdAid); if ((OG->isContained(rdAid) == true) || (OG->isSuspicious(rdAid) == true)) continue; for (uint32 ri=0; ri don't care about this read! // Compute the position (in the tig) of the best overlaps. int32 tig5bgn=0, tig5end=0; int32 tig3bgn=0, tig3end=0; // Instead of using the best edge - which might not be the edge used in the unitig - // we need to scan the layout to return the previous/next dovetail // Put this in a function - what to return if no best overlap? BestEdgeOverlap *b5 = OG->getBestEdgeOverlap(rdAid, false); BestEdgeOverlap *b3 = OG->getBestEdgeOverlap(rdAid, true); // If the best edge is to a read not in this tig, there is nothing to compare against. // Is this confused by default? Possibly. The unitig was constructed somehow, and that // must then be the edge coming into us. We'll pick it up later. bool b5use = true; bool b3use = true; if (b5->fragId() == 0) b5use = false; if (b3->fragId() == 0) b3use = false; if ((b5use) && (Unitig::fragIn(b5->fragId()) != tig->id())) b5use = false; if ((b3use) && (Unitig::fragIn(b3->fragId()) != tig->id())) b3use = false; // The best edge read is in this tig. If they don't overlap, again, nothing to compare // against. if (b5use) { ufNode *rdB = &tig->ufpath[Unitig::pathPosition(b5->fragId())]; uint32 rdBid = rdB->ident; bool rdBfwd = (rdB->position.bgn < rdB->position.end); int32 rdBlo = (rdBfwd) ? rdB->position.bgn : rdB->position.end; int32 rdBhi = (rdBfwd) ? rdB->position.end : rdB->position.bgn; if ((rdAhi < rdBlo) || (rdBhi < rdAlo)) b5use = false; } if (b3use) { ufNode *rdB = &tig->ufpath[Unitig::pathPosition(b3->fragId())]; uint32 rdBid = rdB->ident; bool rdBfwd = (rdB->position.bgn < rdB->position.end); int32 rdBlo = (rdBfwd) ? rdB->position.bgn : rdB->position.end; int32 rdBhi = (rdBfwd) ? rdB->position.end : rdB->position.bgn; if ((rdAhi < rdBlo) || (rdBhi < rdAlo)) b3use = false; } // If we can use this edge, compute the placement of the overlap on the unitig. // Call #1; if (b5use) { int32 bgn=0, end=0; olapToReadCoords(rdA, b5->ahang(), b5->bhang(), bgn, end); tig5bgn = (rdAfwd) ? (rdAlo + sc * bgn) : (rdAhi - sc * end); tig5end = (rdAfwd) ? (rdAlo + sc * end) : (rdAhi - sc * bgn); assert(tig5bgn < tig5end); if (tig5bgn < 0) tig5bgn = 0; if (tig5end > tig->getLength()) tig5end = tig->getLength(); } // Call #2 if (b3use) { int32 bgn=0, end=0; olapToReadCoords(rdA, b3->ahang(), b3->bhang(), bgn, end); tig3bgn = (rdAfwd) ? (rdAlo + sc * bgn) : (rdAhi - sc * end); tig3end = (rdAfwd) ? (rdAlo + sc * end) : (rdAhi - sc * bgn); assert(tig3bgn < tig3end); if (tig3bgn < 0) tig3bgn = 0; if (tig3end > tig->getLength()) tig3end = tig->getLength(); } // If either of the 5' or 3' overlaps (or both!) are in the repeat region, we need to check for // close overlaps on that end. uint32 len5 = 0; uint32 len3 = 0; if ((rMin < tig5bgn) && (tig5end < rMax) && (b5use)) len5 = FI->overlapLength(rdAid, b5->fragId(), b5->ahang(), b5->bhang()); else b5use = false; if ((rMin < tig3bgn) && (tig3end < rMax) && (b3use)) len3 = FI->overlapLength(rdAid, b3->fragId(), b3->ahang(), b3->bhang()); else b3use = false; double score5 = len5 * (1 - b5->erate()); double score3 = len3 * (1 - b3->erate()); // Neither of the best edges are in the repeat region; move to the next region and/or read. if (len5 + len3 == 0) continue; // At least one of the best edge overlaps is in the repeat region. Scan for other edges // that are of comparable length and quality. uint32 ovlLen = 0; BAToverlap *ovl = OC->getOverlaps(rdAid, AS_MAX_ERATE, ovlLen); for (uint32 oo=0; ooufpath.size() == 1)) continue; // If the read is in an annotated bubble, skip. if (unitigs[tgBid]->_isBubble) continue; // Skip if this overlap is the best we're trying to match. if ((rdBid == b5->fragId()) || (rdBid == b3->fragId())) continue; // Skip if this overlap is crappy quality if (OG->isOverlapBadQuality(ovl[oo])) continue; // Skip if the read is contained or suspicious. if ((OG->isContained(rdBid) == true) || (OG->isSuspicious(rdBid) == true)) continue; // Skip if the overlap isn't dovetail. bool ovl5 = ovl[oo].AEndIs5prime(); bool ovl3 = ovl[oo].AEndIs3prime(); if ((ovl5 == false) && (ovl3 == false)) continue; // Skip if we're not using this overlap if ((ovl5 == true) && (b5use == false)) continue; if ((ovl3 == true) && (b3use == false)) continue; uint32 rdBpos = unitigs[tgBid]->pathPosition(rdBid); ufNode *rdB = &unitigs[tgBid]->ufpath[rdBpos]; bool rdBfwd = (rdB->position.bgn < rdB->position.end); int32 rdBlo = (rdBfwd) ? rdB->position.bgn : rdB->position.end; int32 rdBhi = (rdBfwd) ? rdB->position.end : rdB->position.bgn; // If the overlap is to a read in a different tig, or // the overlap is to a read in the same tig, but we don't overlap in the tig, check lengths. // Otherwise, the overlap is present in the tig, and can't be confused. if ((tgBid == tig->id()) && (rdBlo <= rdAhi) && (rdAlo <= rdBhi)) continue; uint32 len = FI->overlapLength(rdAid, ovl[oo].b_iid, ovl[oo].a_hang, ovl[oo].b_hang); double score = len * (1 - ovl[oo].erate); // Compute percent difference. double ad5 = fabs(score - score5); double ad3 = fabs(score - score3); double pd5 = 200 * ad5 / (score + score5); double pd3 = 200 * ad3 / (score + score3); // Skip if this overlap is vastly worse than the best. if ((ovl5 == true) && ((ad5 >= confusedAbsolute) || (pd3 > confusedPercent))) { writeLog("tig %7u read %8u pos %7u-%-7u NOT confused by 5' edge to read %8u - best edge read %8u len %6u erate %.4f score %8.2f - alt edge len %6u erate %.4f score %8.2f - absdiff %8.2f percdiff %8.4f\n", tig->id(), rdAid, rdAlo, rdAhi, rdBid, b5->fragId(), len5, b5->erate(), score5, len, ovl[oo].erate, score, ad5, pd5); continue; } if ((ovl3 == true) && ((ad3 >= confusedAbsolute) || (pd3 > confusedPercent))) { writeLog("tig %7u read %8u pos %7u-%-7u NOT confused by 3' edge to read %8u - best edge read %8u len %6u erate %.4f score %8.2f - alt edge len %6u erate %.4f score %8.2f - absdiff %8.2f percdiff %8.4f\n", tig->id(), rdAid, rdAlo, rdAhi, rdBid, b3->fragId(), len3, b3->erate(), score3, len, ovl[oo].erate, score, ad3, pd3); continue; } // Potential confusion! if (ovl5 == true) writeLog("tig %7u read %8u pos %7u-%-7u IS confused by 5' edge to read %8u - best edge read %8u len %6u erate %.4f score %8.2f - alt edge len %6u erate %.4f score %8.2f - absdiff %8.2f percdiff %8.4f\n", tig->id(), rdAid, rdAlo, rdAhi, rdBid, b5->fragId(), len5, b5->erate(), score5, len, ovl[oo].erate, score, ad5, pd5); if (ovl3 == true) writeLog("tig %7u read %8u pos %7u-%-7u IS confused by 3' edge to read %8u - best edge read %8u len %6u erate %.4f score %8.2f - alt edge len %6u erate %.4f score %8.2f - absdiff %8.2f percdiff %8.4f\n", tig->id(), rdAid, rdAlo, rdAhi, rdBid, b3->fragId(), len3, b3->erate(), score3, len, ovl[oo].erate, score, ad3, pd3); isConfused[ri]++; } } // Over all marks (ri) } // Over all reads (fi) // Scan all the regions, and delete any that have no confusion. { bool discarded = false; for (uint32 ri=0; ri nonRepeatIntervals; writeLog("Scan reads to merge repeat regions.\n"); for (uint32 fi=0; fiufpath.size(); fi++) { ufNode *frg = &tig->ufpath[fi]; bool frgfwd = (frg->position.bgn < frg->position.end); int32 frglo = (frgfwd) ? frg->position.bgn : frg->position.end; int32 frghi = (frgfwd) ? frg->position.end : frg->position.bgn; bool merged = false; for (uint32 ri=1; riident, frglo, frghi); tigMarksR.lo(ri) = tigMarksR.lo(ri-1); tigMarksR.lo(ri-1) = 0; // CRITICAL to delete this interval (and not ri) because the next tigMarksR.hi(ri-1) = 0; // iteration will be using ri-1 (== ri here) and ri (== ri+1). merged = true; nonRepeatIntervals.insert(tigMarksR.lo(ri)); } } if (merged) tigMarksR.filterShort(1); } // Extend the regions by MIN_ANCHOR_HANG. This makes checking for reads that span and are // anchored in the next region easier. It also solved a quirk when the first/last repeat // region doesn't extend to the end of the sequence: // 0-183 unique (created from inversion below, but useless and incorrect) // 183-9942 repeat for (uint32 ii=0; ii(tigMarksR.lo(ii) - MIN_ANCHOR_HANG, 0); tigMarksR.hi(ii) = min(tigMarksR.hi(ii) + MIN_ANCHOR_HANG, tig->getLength()); } // Find the non-repeat intervals. tigMarksU = tigMarksR; tigMarksU.invert(0, tig->getLength()); // Create the list of intervals we'll use to make new unitigs. // // The repeat intervals are extended by MIN_ANCHOR_HANG, and then any read fully contained in one of // these is moved here. // // The non-repeat intervals are shortened by the same amount, and any read that intersects one // is moved there. // // Does order matter? Not sure. The repeat intervals are first, then the formerly repeat // merged intervals, then the unique intervals. Splitting might depend on the repeats being // first. writeLog("Make breakpoints.\n"); vector BP; for (uint32 ii=0; ii 1) splitUnitigs(unitigs, tig, BP, newTigs, lowCoord, nRepeat, nUnique, true); // Report the tigs created. for (uint32 ii=0; ii 1) writeLog("For tig %5u %s region %8d %8d - %6u/%6u reads repeat/unique - unitig %5u created.\n", ti, (repeat == true) ? "repeat" : "unique", rgnbgn, rgnend, nRepeat[ii], nUnique[ii], newTigs[ii]->id()); else writeLog("For tig %5u %s region %8d %8d - %6u/%6u repeat/unique reads - unitig %5u remains unchanged.\n", ti, (repeat == true) ? "repeat" : "unique", rgnbgn, rgnend, nRepeat[ii], nUnique[ii], tig->id()); } // Cleanup. delete [] newTigs; delete [] lowCoord; delete [] nRepeat; delete [] nUnique; // Remove the old unitig....if we made new ones. if (nTigs > 1) { delete tig; unitigs[ti] = NULL; } } } canu-1.3/src/bogart/AS_BAT_MarkRepeatReads.H000066400000000000000000000022661272335422300205550ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAR-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_MARKREPEATREADS #define INCLUDE_AS_BAT_MARKREPEATREADS void markRepeatReads(UnitigVector &unitigs, double deviationRepeat, uint32 confusedAbsolute, double confusedPercent); #endif // INCLUDE_AS_BAT_MARKREPEATREADS canu-1.3/src/bogart/AS_BAT_MergeUnitigs.C000066400000000000000000000233041272335422300201340ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAY-17 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_OverlapCache.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_PlaceFragUsingOverlaps.H" #include "intervalList.H" #include "stddev.H" #include using namespace std; void mergeUnitigs_findPlacements(UnitigVector &unitigs, ufNode *rd, double deviation, vector &validPlacements) { vector placements; placeFragUsingOverlaps(unitigs, AS_MAX_ERATE, NULL, rd->ident, placements); for (uint32 pi=0; piposition.min() < end) && (bgn < rd->position.max())) // Ignore placements to the same place continue; if ((placements[pi].fCoverage < 0.99) || // Ignore partially placed reads. (tig->ufpath.size() == 1)) { // Ignore placements in singletons. //writeLog("read %8u tig %6u (%8u-%8u) placed -- tig %6u (%6u reads) at %8u-%8u (cov %7.5f erate %6.4f) - LOW COV or SINGLETON\n", // rd->ident, Unitig::fragIn(rd->ident), rd->position.bgn, rd->position.end, // placements[pi].tigID, tig->ufpath.size(), placements[pi].position.bgn, placements[pi].position.end, placements[pi].fCoverage, erate); continue; } if (tig->overlapConsistentWithTig(deviation, bgn, end, erate) < 0.5) { //if ((enableLog == true) && (logFileFlagSet(LOG_PLACE_UNPLACED))) // writeLog("read %8u tig %6u (%8u-%8u) placed -- tig %6u (%6u reads) at %8u-%8u (cov %7.5f erate %6.4f) - HIGH ERROR\n", // rd->ident, Unitig::fragIn(rd->ident), rd->position.bgn, rd->position.end, // placements[pi].tigID, tig->ufpath.size(), placements[pi].position.bgn, placements[pi].position.end, placements[pi].fCoverage, erate); continue; } writeLog("read %8u tig %6u (%8u-%8u) placed -- tig %6u (%6u reads) at %8u-%8u (cov %7.5f erate %6.4f)\n", rd->ident, Unitig::fragIn(rd->ident), rd->position.bgn, rd->position.end, placements[pi].tigID, tig->ufpath.size(), placements[pi].position.bgn, placements[pi].position.end, placements[pi].fCoverage, erate); validPlacements.push_back(placements[pi]); } } void mergeUnitigs(UnitigVector &unitigs, double deviation, bool findCircularTigs) { // For every tig, decide if it can merge, end-to-end, with some other tig. This operation // should occur before bubbles are popped (so that whatever we chop off can be popped as a // bubble) and repeats are split (so that whatever we join can be split if it's not supported). // // The basic idea is that the end read on each tig should align to the middle of the other tig. // If the reads between those also align, we can merge. If they do not align, we should split // off one end and join. The split off end is either a bubble, or we made bad joins and will // end up with four pieces after repeat breaking. // // ----------------------------------- // ^^^^ ---- // |||| |||| // ---- vvvv // ---------------------------------- // // This is the same basic operation as for finding circular tigs, and those // are found too. However, this should occur after bubbles and repeats. // Step 1: For every end read, place it. Save only placements that are full-length and // compatible with the destination tig. vector validPlacements; for (uint32 ti=0; tigetNumFrags() < 2) || (tig->_isUnassembled == true)) continue; ufNode *f = tig->firstRead(); ufNode *l = tig->lastRead(); if (f == l) continue; mergeUnitigs_findPlacements(unitigs, f, deviation, validPlacements); mergeUnitigs_findPlacements(unitigs, l, deviation, validPlacements); } writeLog("Found "F_SIZE_T" valid placements of end reads.\n", validPlacements.size()); // Step 2: Find pairs of placements between two tigs. vector > potentialCircles; vector > potentialMerges; for (uint32 pa=0; pa(pa, pb)); continue; } if ((paDstTigID == pbSrcTigID) && // pa placed in same tig as pb came from (pbDstTigID == paSrcTigID)) { // pb placed in same tig as pa came from potentialMerges.push_back(pair(pa, pb)); continue; } } } writeLog("Found "F_SIZE_T" potential circular tigs.\n", potentialCircles.size()); writeLog("Found "F_SIZE_T" potential joins.\n", potentialMerges.size()); // Step 3: For the potential circles, each read needs to be placed with the same orientation as // its source, and the distance between (paSrc,pbDst) and (pbSrc,paDst) needs to be // (approximately) the same. Then, we should really check the reads between those two points. for (uint32 pc=0; pcufpath[Unitig::pathPosition(paReadID)]; bool paSrcFwd = paRead->position.isForward(); bool paDstFwd = validPlacements[pa].position.isForward(); ufNode *pbRead = &tig->ufpath[Unitig::pathPosition(pbReadID)]; bool pbSrcFwd = pbRead->position.isForward(); bool pbDstFwd = validPlacements[pb].position.isForward(); writeLog("TEST CIRCULAR - tig %u - pa=%u pb=%u - reads %u @ %u-%u -> %u-%u and %u @ %u-%u -> %u-%u\n", tigID, pa, pb, paReadID, paRead->position.bgn, paRead->position.end, validPlacements[pa].position.bgn, validPlacements[pa].position.end, pbReadID, pbRead->position.bgn, pbRead->position.end, validPlacements[pb].position.bgn, validPlacements[pb].position.end); if ((paSrcFwd != paDstFwd) || (pbSrcFwd != pbDstFwd)) { writeLog("not circular - orient mismatch for tig %u pa %u pb %u reads %u and %u\n", tigID, pa, pb, paReadID, pbReadID); continue; } } for (uint32 pc=0; pcufpath[Unitig::pathPosition(paReadID)]; bool paSrcFwd = paRead->position.isForward(); bool paDstFwd = validPlacements[pa].position.isForward(); ufNode *pbRead = &pbTig->ufpath[Unitig::pathPosition(pbReadID)]; bool pbSrcFwd = pbRead->position.isForward(); bool pbDstFwd = validPlacements[pb].position.isForward(); writeLog("TEST JOIN - pa tig %u read %u @ %u-%u -> %u-%u -- pb tig %u read %u @ %u-%u -> %u-%u\n", paTigID, paReadID, paRead->position.bgn, paRead->position.end, validPlacements[pa].position.bgn, validPlacements[pa].position.end, pbTigID, pbReadID, pbRead->position.bgn, pbRead->position.end, validPlacements[pb].position.bgn, validPlacements[pb].position.end); } exit(0); } canu-1.3/src/bogart/AS_BAT_MergeUnitigs.H000066400000000000000000000021611272335422300201370ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAY-17 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_MERGE_UNITIGS #define INCLUDE_AS_BAT_MERGE_UNITIGS void mergeUnitigs(UnitigVector &unitigs, double deviation, bool findCircularTigs); #endif // INCLUDE_AS_BAT_MERGE_UNITIGS canu-1.3/src/bogart/AS_BAT_Outputs.C000066400000000000000000000365031272335422300172220ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_Outputs.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2014-MAR-31 * are Copyright 2010-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-JUN-05 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-04 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-MAR-30 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_OverlapCache.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_PlaceFragUsingOverlaps.H" #include "tgStore.H" void unitigToTig(tgTig *tig, uint32 tigid, Unitig *utg) { // Initialize the output tig. tig->clear(); tig->_tigID = tigid; utg->_tigID = tigid; tig->_coverageStat = 1.0; // Default to just barely unique tig->_microhetProb = 1.0; // Default to 100% probability of unique // Set the class. if (utg->_isUnassembled == true) tig->_class = tgTig_unassembled; else if (utg->_isBubble == true) tig->_class = tgTig_bubble; else tig->_class = tgTig_contig; tig->_suggestRepeat = (utg->_isRepeat == true); tig->_suggestCircular = (utg->_isCircular == true); tig->_layoutLen = utg->getLength(); // Transfer reads from the bogart tig to the output tig. resizeArray(tig->_children, tig->_childrenLen, tig->_childrenMax, utg->ufpath.size(), resizeArray_doNothing); for (uint32 ti=0; tiufpath.size(); ti++) { ufNode *frg = &utg->ufpath[ti]; tig->addChild()->set(frg->ident, frg->parent, frg->ahang, frg->bhang, frg->position.bgn, frg->position.end); } } void writeUnitigsToStore(UnitigVector &unitigs, char *fileprefix, char *tigStorePath, uint32 frg_count_target, bool isFinal) { uint32 utg_count = 0; uint32 frg_count = 0; uint32 prt_count = 1; char filename[FILENAME_MAX] = {0}; // Open up the initial output file sprintf(filename, "%s.iidmap", fileprefix); FILE *iidm = fopen(filename, "w"); assert(NULL != iidm); sprintf(filename, "%s.partitioning", fileprefix); FILE *part = fopen(filename, "w"); assert(NULL != part); sprintf(filename, "%s.partitioningInfo", fileprefix); FILE *pari = fopen(filename, "w"); assert(NULL != pari); // Step through all the unitigs once to build the partition mapping and IID mapping. tgStore *tigStore = new tgStore(tigStorePath); tgTig *tig = new tgTig; for (uint32 tigID=0, ti=0; tigetNumFrags() == 0)) continue; assert(utg->getLength() > 0); // Convert the bogart tig to a tgTig and save to the store. unitigToTig(tig, (isFinal) ? tigID : ti, utg); tigID++; tigStore->insertTig(tig, false); // Increment the partition if the current one is too large. if ((frg_count + utg->getNumFrags() >= frg_count_target) && (frg_count > 0)) { fprintf(pari, "Partition %d has %d unitigs and %d fragments.\n", prt_count, utg_count, frg_count); prt_count++; utg_count = 0; frg_count = 0; } // Note that the tig is included in this partition. utg_count += 1; frg_count += utg->getNumFrags(); // Map the tig to a partition, and log both the tig-to-partition map and the partition-to-read map. fprintf(iidm, "bogart "F_U32" -> tig "F_U32" (in partition "F_U32" with "F_U32" frags)\n", utg->id(), utg->tigID(), prt_count, utg->getNumFrags()); for (uint32 fragIdx=0; fragIdxgetNumFrags(); fragIdx++) fprintf(part, "%d\t%d\n", prt_count, utg->ufpath[fragIdx].ident); } fprintf(pari, "Partition %d has %d unitigs and %d fragments.\n", // Don't forget to log the last partition! prt_count, utg_count, frg_count); fclose(pari); fclose(part); fclose(iidm); delete tig; delete tigStore; } class rawEdge_t { public: rawEdge_t(uint32 o, uint32 t, int32 ab, int32 ae, int32 bb, int32 be) { oi = o; tigID = t; Abgn = ab; Aend = ae; Bbgn = bb; Bend = be; }; uint32 oi; int32 tigID; int32 Abgn; // Overlapping read placement. int32 Aend; int32 Bbgn; // Parent placement. int32 Bend; bool operator<(rawEdge_t const &that) const { if (tigID != that.tigID) return(tigID < that.tigID); return(Abgn < that.Abgn); } }; void findUnusedEdges(UnitigVector &unitigs, ufNode *rdA, // Read we're finding edges for bool rdA3p, // Overlaps from the 3' end of the read set edgeReads, FILE *EF) { uint32 rdAid = rdA->ident; uint32 rdAlen = FI->fragmentLength(rdAid); bool rdAfwd = rdA->isForward(); int32 rdAlo = (rdAfwd) ? (rdA->position.bgn) : (rdA->position.end); int32 rdAhi = (rdAfwd) ? (rdA->position.end) : (rdA->position.bgn); uint32 rdAtigID = Unitig::fragIn(rdAid); Unitig *rdAtig = unitigs[rdAtigID]; uint32 ovlLen = 0; BAToverlap *ovl = OC->getOverlaps(rdA->ident, AS_MAX_ERATE, ovlLen); vector rawEdges; //fprintf(stderr, "WORKING ON read rdA=%u 3p=%d\n", rdA->ident, rdA3p); // Over all overlaps for this read, find and report edges to 'edgeReads'. Though // edgeReads should be just one read per tig end, the code below was originally written // to find all edges to all reads, then pick the longest for each cluster. for (uint32 oi=0; oigetNumFrags() == 0) || // Not interested in edges to singletons (rdBtig->_isUnassembled == true)) // Or other unassembled crap. rdA filtered outside here. continue; if ((rdAtigID != rdBtigID) && // Not to self (circular) and (edgeReads.count(rdBid) == 0)) // not a read we can overlap to. continue; ufNode *rdB = &rdBtig->ufpath[ Unitig::pathPosition(rdBid) ]; bool rdBfwd = rdB->isForward(); int32 rdBlo = (rdBfwd) ? (rdB->position.bgn) : (rdB->position.end); int32 rdBhi = (rdBfwd) ? (rdB->position.end) : (rdB->position.bgn); // Exclude overlaps satisfied in the same tig. if ((rdAtigID == rdBtigID) && (rdAlo < rdBhi) && (rdBlo < rdAhi)) continue; // Exclude overlaps that are higher than expected error. ; // Compute the placement of rdA on rdBtig. ufNode placed; BestEdgeOverlap edge(ovl[oi]); rdBtig->placeFrag(placed, rdAid, rdA3p, &edge); //writeLog("placed tig %u rdA %u %d-%d on tig %u %d-%d from rdB %u %d-%d oi %u\n", // rdAtigID, rdAid, rdAlo, rdAhi, rdBtigID, placed.position.bgn, placed.position.end, rdBid, rdBlo, rdBhi, oi); // Save the overlap. rawEdges.push_back(rawEdge_t(oi, rdBtigID, placed.position.min(), placed.position.max(), rdBlo, rdBhi)); } // We've now got a pile of (unsorted) overlaps to reads in other tigs. We need to pick one // overlap (the longest?) from each pile and output it. sort(rawEdges.begin(), rawEdges.end()); // We expect to have a pile of placements that are 'the same', generated by each one of the // overlapping reads in the target tig. We need to group these placements together and pick // one exemplar overlap to output the edge for. // // A complication is caused by large tandem repeats. We can get two distinct placements that // overlap: // // [rrrr][rrrr] // --------------- (rdA aligning to the first and second repeat) // ---------------- (rdA aligning to only the second repeat) // // These are just overlaps, and we don't know that the rest of rdA fails to align. // // Overlaps are sorted by the start of rdA on rdBtig. We'll use the simple and largely unvalidated // heuristic of any placement that starts within 500bp of the last is for the same placement. for (uint32 ri=0, rj=0; ri= rawEdges[rj].Abgn)); ) rj++; // Scan overlaps from ri to rj, retain the thickest. //fprintf(stderr, "Scan batch from ri=%u to rj=%u\n", ri, rj); uint32 rrMax = 0; int32 rrIdx = INT32_MAX; for (uint32 rr=ri; rr= rawEdges[rr].Abgn); olapLen = rawEdges[rr].Bend - rawEdges[rr].Abgn; } else { assert(rawEdges[rr].Aend >= rawEdges[rr].Bbgn); olapLen = rawEdges[rr].Aend - rawEdges[rr].Bbgn; } if (rrMax < olapLen) { rrMax = olapLen; rrIdx = rr; } } // Emit the edge. uint32 oi = rawEdges[rrIdx].oi; uint32 rdBid = ovl[oi].b_iid; uint32 rdBtigID = Unitig::fragIn(rdBid); Unitig *rdBtig = unitigs[rdBtigID]; ufNode *rdB = &rdBtig->ufpath[ Unitig::pathPosition(rdBid) ]; bool rdBfwd = rdB->isForward(); int32 rdBlo = (rdBfwd) ? (rdB->position.bgn) : (rdB->position.end); int32 rdBhi = (rdBfwd) ? (rdB->position.end) : (rdB->position.bgn); char rdAEnd, rdBEnd; if (ovl[oi].isDovetail()) { rdAEnd = ovl[oi].AEndIs5prime() ? '5' : '3'; rdBEnd = ovl[oi].BEndIs5prime() ? '5' : '3'; } else { rdAEnd = ovl[oi].AisContainer() ? 'C' : 'c'; rdBEnd = ovl[oi].AisContainer() ? 'c' : 'C'; } char ori = (ovl[oi].flipped) ? '<' : '>'; fprintf(EF, "tig %7u %c read %8u at %9u %-9u %8d %c %-8d tig %7u %c read %8u at %9u %-9u\n", rdAtig->_tigID, rdAtig->type(), rdAid, rdA->position.bgn, rdA->position.end, ovl[oi].a_hang, ori, ovl[oi].b_hang, rdBtig->_tigID, rdBtig->type(), rdBid, rdB->position.bgn, rdB->position.end); } } void writeUnusedEdges(UnitigVector &unitigs, char *fileprefix) { char filename[FILENAME_MAX] = {0}; sprintf(filename, "%s.unused.edges", fileprefix); FILE *EF = fopen(filename, "w"); if (errno) fprintf(stderr, "Failed to create unused edge output '%s': %s\n", filename, strerror(errno)), exit(1); // Find reads we're allowed to find edges to. We can pick either the outer-most non-contained reads, // or just the reads touching the edge of the tig. set edgeReads; // Reads at the end of the tig set nearReads; // Reads close to the end of the tig // Find the outer-most non-contained reads in each unitig. #if 0 for (uint32 ti=0; tigetNumFrags() == 0) || (tig->_isUnassembled == true)) continue; // Find reads at the start of the tig for (uint32 ct=0, fi=0; (ct < 5) && (fi < tig->ufpath.size()); fi++) { ufNode *frg = &tig->ufpath[fi]; if (OG->isContained(frg->ident) == false) { if (ct == 0) edgeReads5e.insert(frg->ident); else nearReads5m.insert(frg->ident); ct++; } } // Find reads at the end of the tig for (uint32 ct=0, fi=tig->ufpath.size(); (ct < 5) && (fi-- > 0); ) { ufNode *frg = &tig->ufpath[fi]; if (OG->isContained(frg->ident) == false) { if (ct == 0) edgeReads3e.insert(frg->ident); else nearReads3m.insert(frg->ident); ct++; } } } #endif // Find the reads at the ends of the tig. #if 1 for (uint32 ti=0; tigetNumFrags() == 0) || (tig->_isUnassembled == true)) continue; edgeReads.insert(tig->firstRead()->ident); edgeReads.insert(tig->lastRead()->ident); } #endif // Step through all the unitigs, find all unused overlaps off the ends of the tig. for (uint32 ti=0; tigetNumFrags() == 0) || (tig->_isUnassembled == true)) continue; assert(tig->getLength() > 0); // Find the first/last non-contained reads in the tig. #if 0 ufNode *rd5 = &tig->ufpath.front(); ufNode *rd3 = &tig->ufpath.back(); for (uint32 fi=1; (fi < tig->ufpath.size()) && (OG->isContained(rd5->ident) == true); fi++) rd5 = &tig->ufpath[fi]; for (uint32 fi=tig->ufpath.size()-1; (fi-- > 0) && (OG->isContained(rd3->ident) == true); ) rd3 = &tig->ufpath[fi]; // What to do if either of those reads are contained? If so (then both will be contained; no // dovetail at all) we've swapped the meaning of 5' and 3'. if ((OG->isContained(rd5) == true) || (OG->isContained(rd3) == true)) { rd5 = &tig->ufpath.front(); rd3 = &tig->ufpath.back(); } #endif // Find the smallest/largest read position - the two reads that are at the end of the tig. #if 1 ufNode *rd5 = tig->firstRead(); ufNode *rd3 = tig->lastRead(); #endif // Finally, we probably should be finding just the reads touching the ends of the unitig, not the // first/last non-contained read. findUnusedEdges(unitigs, rd5, rd5->isReverse(), edgeReads, EF); // First read, if reverse, find edges off 3' end findUnusedEdges(unitigs, rd3, rd3->isForward(), edgeReads, EF); // Last read, if forward, find edges off 3' end } fclose(EF); } canu-1.3/src/bogart/AS_BAT_Outputs.H000066400000000000000000000034361272335422300172260ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_Outputs.H * * Modifications by: * * Brian P. Walenz from 2010-DEC-06 to 2013-AUG-01 * are Copyright 2010-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2014-DEC-23 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_OUTPUTS #define INCLUDE_AS_BAT_OUTPUTS #include "AS_BAT_Unitig.H" #include "AS_BAT_UnitigVector.H" void writeUnitigsToStore(UnitigVector &unitigs, char *fileprefix, char *tigStorePath, uint32 frg_count_target, bool isFinal); void writeUnusedEdges(UnitigVector &unitigs, char *fileprefix); #endif // INCLUDE_AS_BAT_OUTPUTS canu-1.3/src/bogart/AS_BAT_OverlapCache.C000066400000000000000000000754101272335422300200730ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_OverlapCache.C * * Modifications by: * * Brian P. Walenz from 2011-FEB-15 to 2013-OCT-14 * are Copyright 2011-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2012-JAN-11 * are Copyright 2012 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz from 2014-AUG-06 to 2015-JUN-25 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-APR-26 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_OverlapCache.H" #include "AS_BAT_BestOverlapGraph.H" // sizeof(BestEdgeOverlap) #include "AS_BAT_Unitig.H" // sizeof(ufNode) #include "AS_BAT_Logging.H" #include "memoryMappedFile.H" #include uint64 ovlCacheMagic = 0x65686361436c766fLLU; //0102030405060708LLU; #ifndef __CYGWIN__ #ifndef _WIN32 #include #endif #endif #ifdef HW_PHYSMEM uint64 getMemorySize(void) { uint64 physMemory = 0; int mib[2] = { CTL_HW, HW_PHYSMEM }; size_t len = sizeof(uint64); errno = 0; if (sysctl(mib, 2, &physMemory, &len, NULL, 0) != 0) // failed to get memory size, so what? fprintf(stderr, "sysctl() failed to return CTL_HW, HW_PHYSMEM: %s\n", strerror(errno)), exit(1); if (len != sizeof(uint64)) { #ifdef HW_MEMSIZE mib[1] = HW_MEMSIZE; len = sizeof(uint64); if (sysctl(mib, 2, &physMemory, &len, NULL, 0) != 0 || len != sizeof(uint64)) #endif // wasn't enough space, so what? fprintf(stderr, "sysctl() failed to return CTL_HW, HW_PHYSMEM: %s\n", strerror(errno)), exit(1); } return(physMemory); } #else uint64 getMemorySize(void) { uint64 physPages = sysconf(_SC_PHYS_PAGES); uint64 pageSize = sysconf(_SC_PAGESIZE); uint64 physMemory = physPages * pageSize; fprintf(stderr, "PHYS_PAGES = "F_U64"\n", physPages); fprintf(stderr, "PAGE_SIZE = "F_U64"\n", pageSize); fprintf(stderr, "MEMORY = "F_U64"\n", physMemory); return(physMemory); } #endif OverlapCache::OverlapCache(ovStore *ovlStoreUniq, ovStore *ovlStoreRept, const char *prefix, double erate, uint32 minOverlap, uint64 memlimit, uint32 maxOverlaps, bool onlySave, bool doSave) { _memLimit = 0; _memUsed = 0; _storMax = 0; _storLen = 0; _stor = NULL; _heaps.clear(); _cacheMMF = NULL; _cachePtr = NULL; _cacheLen = NULL; _maxPer = 0; _ovsMax = 0; _ovs = NULL; _ovsSco = NULL; _ovsTmp = NULL; _threadMax = 0; _thread = NULL; _ovlStoreUniq = NULL; _ovlStoreRept = NULL; if (load(prefix, erate) == true) return; fprintf(stderr, "\n"); if (memlimit == UINT64_MAX) { _memLimit = getMemorySize(); fprintf(stderr, "OverlapCache()-- limited to "F_U64"MB memory (total physical memory).\n", _memLimit >> 20); } else if (memlimit > 0) { _memLimit = memlimit; fprintf(stderr, "OverlapCache()-- limited to "F_U64"MB memory (user supplied).\n", _memLimit >> 20); } else { fprintf(stderr, "OverlapCache()-- using unlimited memory (-M 0).\n"); _memLimit = UINT64_MAX; } // Need to initialize thread data before we can account for their size. _threadMax = omp_get_max_threads(); _thread = new OverlapCacheThreadData [_threadMax]; // And this too. _ovsMax = 1 * 1024 * 1024; // At 16B each, this is 16MB // Account for memory used by fragment data, best overlaps, and unitigs. // The chunk graph is temporary, and should be less than the size of the unitigs. uint64 memFI = FI->memoryUsage(); uint64 memBE = FI->numFragments() * sizeof(BestEdgeOverlap); uint64 memUL = FI->numFragments() * sizeof(ufNode); // For fragment positions in unitigs uint64 memUT = FI->numFragments() * sizeof(uint32) / 16; // For unitigs (assumes 32 frag / unitig) uint64 memID = FI->numFragments() * sizeof(uint32) * 2; // For maps of fragment id to unitig id uint64 memEP = FI->numFragments() * Unitig::epValueSize() * 2; // For error profile uint64 memC1 = (FI->numFragments() + 1) * (sizeof(BAToverlapInt *) + sizeof(uint32)); uint64 memC2 = _ovsMax * (sizeof(ovOverlap) + sizeof(uint64) + sizeof(uint64)); uint64 memC3 = _threadMax * _thread[0]._batMax * sizeof(BAToverlap); uint64 memC4 = (FI->numFragments() + 1) * sizeof(uint32); uint64 memOS = (_memLimit == getMemorySize()) ? (0.1 * getMemorySize()) : 0.0; uint64 memTT = memFI + memBE + memUL + memUT + memID + memC1 + memC2 + memC3 + memC4 + memOS; if (onlySave) { fprintf(stderr, "OverlapCache()-- Only saving overlaps, not computing unitigs.\n"); memBE = 0; memUL = 0; memUT = 0; memID = 0; memTT = memFI + memBE + memUL + memUT + memID + memOS + memC1 + memC2 + memC3 + memC4; } fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for fragment data.\n", memFI >> 20); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for best edges.\n", memBE >> 20); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for unitig layouts.\n", memUL >> 20); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for unitigs.\n", memUT >> 20); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for id maps.\n", memID >> 20); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for error profiles.\n", memEP >> 20); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for overlap cache pointers.\n", memC1 >> 20); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for overlap cache initial bucket.\n", memC2 >> 20); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for overlap cache thread data.\n", memC3 >> 20); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for number of overlaps per read.\n", memC4 >> 20); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for other processes.\n", memOS >> 20); fprintf(stderr, "OverlapCache()-- ---------\n"); fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB for data structures (sum of above).\n", memTT >> 20); if (_memLimit <= memTT) { int64 defecit = (int64)memTT - (int64)_memLimit; fprintf(stderr, "OverlapCache()-- %7"F_S64P"MB available for overlaps.\n", defecit); fprintf(stderr, "OverlapCache()-- Out of memory before loading overlaps; increase -M.\n"); exit(1); } _memLimit -= memTT; _memUsed = 0; fprintf(stderr, "OverlapCache()-- %7"F_U64P"MB available for overlaps.\n", _memLimit >> 20); fprintf(stderr, "\n"); // Decide on the default block size. We want to use large blocks (to reduce the number of // allocations, and load on the allocator) but not so large that we can't fit nicely. // // 1gb blocks @ 64 -> 64gb // 128mb blocks @ 64 -> 8gb // // below 8gb we'll use 128mb blocks // from 8gb to 64gb, we'll use _memLimit/64 // from 64gb on, we'll use 1gb block if (_memLimit <= (uint64)8 * 1024 * 1024 * 1024) _storMax = 128 * 1024 * 1024 / sizeof(BAToverlapInt); else if (_memLimit <= (uint64)64 * 1024 * 1024 * 1024) _storMax = _memLimit / 64 / sizeof(BAToverlapInt); else _storMax = (uint64)1024 * 1024 * 1024 / sizeof(BAToverlapInt); _storLen = 0; _stor = NULL; _cacheMMF = NULL; _cachePtr = new BAToverlapInt * [FI->numFragments() + 1]; _cacheLen = new uint32 [FI->numFragments() + 1]; memset(_cachePtr, 0, sizeof(BAToverlapInt *) * (FI->numFragments() + 1)); memset(_cacheLen, 0, sizeof(uint32) * (FI->numFragments() + 1)); _maxPer = maxOverlaps; _ovs = ovOverlap::allocateOverlaps(NULL, _ovsMax); // So can't call bgn or end. _ovsSco = new uint64 [_ovsMax]; _ovsTmp = new uint64 [_ovsMax]; _ovlStoreUniq = ovlStoreUniq; _ovlStoreRept = ovlStoreRept; assert(_ovlStoreUniq != NULL); assert(_ovlStoreRept == NULL); if (_memUsed > _memLimit) fprintf(stderr, "OverlapCache()-- ERROR: not enough memory to load ANY overlaps.\n"), exit(1); computeOverlapLimit(); loadOverlaps(erate, minOverlap, prefix, onlySave, doSave); delete [] _ovs; _ovs = NULL; delete [] _ovsSco; _ovsSco = NULL; delete [] _ovsTmp; _ovsTmp = NULL; if (doSave == true) save(prefix, erate); if ((doSave == true) && (onlySave == true)) fprintf(stderr, "Exiting; only requested to build the overlap graph.\n"), exit(0); } OverlapCache::~OverlapCache() { if (_cacheMMF) { _stor = NULL; delete _cacheMMF; } delete [] _ovs; delete [] _thread; delete [] _cacheLen; delete [] _cachePtr; for (uint32 i=0; i<_heaps.size(); i++) delete [] _heaps[i]; } // Decide on limits per fragment. // // From the memory limit, we can compute the average allowed per fragment. If this is higher than // the expected coverage, we'll not fill memory completely as the fragments in unique sequence will // have fewer than this number of overlaps. // // We'd like to iterate this, but the unused space computation assumes all fragments are assigned // the same amount of memory. On the next iteration, this isn't true any more. The benefit is // (hopefully) small, and the algorithm is unknown. // // This isn't perfect. It estimates based on whatever is in the store, not only those overlaps // below the error threshold. Result is that memory usage is far below what it should be. Easy to // fix if we assume all fragments have the same properties (same library, same length, same error // rate) but not so easy in reality. We need big architecture changes to make it easy (grouping // reads by library, collecting statistics from the overlaps, etc). // // It also doesn't distinguish between 5' and 3' overlaps - it is possible for all the long // overlaps to be off of one end. // void OverlapCache::computeOverlapLimit(void) { if (_maxPer < UINT32_MAX) { // -N supplied on the command line, use that instead. fprintf(stderr, "OverlapCache()-- _maxPer = "F_U32" overlaps/frag (from command line)\n", _maxPer); return; } _ovlStoreUniq->resetRange(); // AS_OVS_numOverlapsPerFrag returns an array that starts at firstIIDrequested. This is usually // 1, unless the first fragment has no overlaps. In that case, firstIIDrequested will be the // first fragment with overlaps. This is a terrible interface. fprintf(stderr, "OverlapCache()-- Loading number of overlaps per fragment.\n"); uint32 frstFrag = 0; uint32 lastFrag = 0; uint32 *numPer = _ovlStoreUniq->numOverlapsPerFrag(frstFrag, lastFrag); uint32 totlFrag = lastFrag - frstFrag + 1; uint32 numPerMax = 0; for (uint32 i=0; inumFragments() * sizeof(BAToverlapInt)); fprintf(stderr, "OverlapCache()-- Initial guess at _maxPer="F_U32" (max of "F_U32") from (memLimit="F_U64" - memUsed="F_U64") / (numFrags="F_U32" * sizeof(OVL)="F_SIZE_T")\n", _maxPer, numPerMax, _memLimit, _memUsed, FI->numFragments(), sizeof(BAToverlapInt)); if (_maxPer < 10) fprintf(stderr, "OverlapCache()-- ERROR: not enough memory to load overlaps (_maxPer="F_U32" < 10).\n", _maxPer), exit(1); uint64 totalLoad = 0; // Total overlaps we would load at this threshold uint32 numBelow = 0; // Number below the threshold //uint64 numBelowS = 0; // Amount of space wasted beacuse of this uint32 numEqual = 0; uint32 numAbove = 0; // Number of fragments above the threshold uint32 lastMax = 0; uint32 adjust = 1; while (adjust > 0) { totalLoad = 0; numBelow = 0; //numBelowS = 0; numEqual = 0; numAbove = 0; for (uint32 i=0; i> 20, adjust); if (_maxPer > numPerMax) _maxPer = numPerMax; } else { // Whoops! Too high! Revert to the last and recompute statistics. adjust = 0; _maxPer = lastMax; totalLoad = 0; numBelow = 0; numEqual = 0; numAbove = 0; for (uint32 i=0; i> 20); fprintf(stderr, "\n"); fprintf(stderr, "OverlapCache()-- _maxPer = "F_U32" overlaps/reads\n", _maxPer); fprintf(stderr, "OverlapCache()-- numBelow = "F_U32" reads (all overlaps loaded)\n", numBelow); fprintf(stderr, "OverlapCache()-- numEqual = "F_U32" reads (all overlaps loaded)\n", numEqual); fprintf(stderr, "OverlapCache()-- numAbove = "F_U32" reads (some overlaps loaded)\n", numAbove); fprintf(stderr, "OverlapCache()-- totalLoad = "F_U64" overlaps (%6.2f%%)\n", totalLoad, 100.0 * totalLoad / _ovlStoreUniq->numOverlapsInRange()); fprintf(stderr, "\n"); fprintf(stderr, "OverlapCache()-- availForOverlaps = "F_U64"MB\n", _memLimit >> 20); fprintf(stderr, "OverlapCache()-- totalMemory = "F_U64"MB for organization\n", _memUsed >> 20); fprintf(stderr, "OverlapCache()-- totalMemory = "F_U64"MB for overlaps\n", (totalLoad * sizeof(BAToverlapInt)) >> 20); fprintf(stderr, "OverlapCache()-- totalMemory = "F_U64"MB used\n", (_memUsed + totalLoad * sizeof(BAToverlapInt)) >> 20); fprintf(stderr, "\n"); delete [] numPer; } uint32 OverlapCache::filterOverlaps(uint32 maxEvalue, uint32 minOverlap, uint32 no) { uint32 ns = 0; // Score the overlaps. uint64 ERR_MASK = ((uint64)1 << AS_MAX_EVALUE_BITS) - 1; uint32 SALT_BITS = (64 - AS_MAX_READLEN_BITS - AS_MAX_EVALUE_BITS); uint64 SALT_MASK = (((uint64)1 << SALT_BITS) - 1); memset(_ovsSco, 0, sizeof(uint64) * no); for (uint32 ii=0; iifragmentLength(_ovs[ii].a_iid) == 0) || (FI->fragmentLength(_ovs[ii].b_iid) == 0)) // At least one read deleted in the overlap continue; if (_ovs[ii].evalue() > maxEvalue) // Too noisy. continue; uint32 olen = FI->overlapLength(_ovs[ii].a_iid, _ovs[ii].b_iid, _ovs[ii].a_hang(), _ovs[ii].b_hang()); if (olen < minOverlap) // Too short. continue; // Just right! _ovsSco[ii] = olen; _ovsSco[ii] <<= AS_MAX_EVALUE_BITS; _ovsSco[ii] |= (~_ovs[ii].evalue()) & ERR_MASK; _ovsSco[ii] <<= SALT_BITS; _ovsSco[ii] |= ii & SALT_MASK; ns++; } // If fewer than the limit, keep them all. Should we reset ovsSco to be 1? Do we really need ovsTmp? memcpy(_ovsTmp, _ovsSco, sizeof(uint64) * no); if (ns <= _maxPer) return(ns); // Otherwise, filter out the short and low quality. sort(_ovsTmp, _ovsTmp + no); uint64 cutoff = _ovsTmp[no - _maxPer]; for (uint32 ii=0; ii 0) ns++; if (ns > _maxPer) fprintf(stderr, "WARNING: fragment "F_U32" loaded "F_U32" overlas (it has "F_U32" in total); over the limit of "F_U32"\n", _ovs[0].a_iid, ns, no, _maxPer); return(ns); } void OverlapCache::loadOverlaps(double erate, uint32 minOverlap, const char *prefix, bool onlySave, bool doSave) { uint64 numTotal = 0; uint64 numLoaded = 0; uint32 numFrags = 0; uint32 numOvl = 0; uint32 maxEvalue = AS_OVS_encodeEvalue(erate); FILE *ovlDat = NULL; if (doSave == true) { char name[FILENAME_MAX]; sprintf(name, "%s.ovlCacheDat", prefix); fprintf(stderr, "OverlapCache()-- Saving overlaps to '%s'.\n", name); errno = 0; ovlDat = fopen(name, "w"); if (errno) fprintf(stderr, "OverlapCache()-- Failed to open '%s' for write: %s\n", name, strerror(errno)), exit(1); } assert(_ovlStoreUniq != NULL); assert(_ovlStoreRept == NULL); _ovlStoreUniq->resetRange(); uint64 numStore = _ovlStoreUniq->numOverlapsInRange(); writeLog("OverlapCache()-- Loading overlap information\n"); // Could probably easily extend to multiple stores. Needs to interleave the two store // loads, can't do one after the other as we require all overlaps for a single fragment // be in contiguous memory. while (1) { // Ask the store how many overlaps exist for this fragment. numOvl = _ovlStoreUniq->numberOfOverlaps(); numTotal += numOvl; if (numOvl == 0) // No overlaps? We're at the end of the store. break; // Resize temporary storage space to hold all these overlaps. while (_ovsMax <= numOvl) { _memUsed -= (_ovsMax) * sizeof(ovOverlap); _memUsed -= (_ovsMax) * sizeof(uint64); _ovsMax *= 2; delete [] _ovs; delete [] _ovsSco; delete [] _ovsTmp; _ovs = ovOverlap::allocateOverlaps(NULL, _ovsMax); // So can't call bgn or end. _ovsSco = new uint64 [_ovsMax]; _ovsTmp = new uint64 [_ovsMax]; _memUsed += (_ovsMax) * sizeof(ovOverlap); _memUsed += (_ovsMax) * sizeof(uint64); _memUsed += (_ovsMax) * sizeof(uint64); } // Actually load the overlaps. uint32 no = _ovlStoreUniq->readOverlaps(_ovs, _ovsMax); uint32 ns = filterOverlaps(maxEvalue, minOverlap, no); // Resize the permament storage space for overlaps. if ((_storLen + ns > _storMax) || (_stor == NULL)) { if ((ovlDat) && (_storLen > 0)) AS_UTL_safeWrite(ovlDat, _stor, "_stor", sizeof(BAToverlapInt), _storLen); if (onlySave) delete [] _stor; _storLen = 0; _stor = new BAToverlapInt [_storMax]; _heaps.push_back(_stor); _memUsed += _storMax * sizeof(BAToverlapInt); } // Save a pointer to the start of the overlaps for this fragment, and the number of overlaps // that exist. _cachePtr[_ovs[0].a_iid] = _stor + _storLen; _cacheLen[_ovs[0].a_iid] = ns; numLoaded += ns; uint32 storEnd = _storLen + ns; // Finally, append the overlaps to the storage. for (uint32 ii=0; ii 0)) AS_UTL_safeWrite(ovlDat, _stor, "_stor", sizeof(BAToverlapInt), _storLen); if (onlySave) delete [] _stor; if (ovlDat) fclose(ovlDat); writeLog("OverlapCache()-- Loading overlap information: overlaps processed %12"F_U64P" (%06.2f%%) loaded %12"F_U64P" (%06.2f%%)\n", numTotal, 100.0 * numTotal / numStore, numLoaded, 100.0 * numLoaded / numStore); } BAToverlap * OverlapCache::getOverlaps(uint32 fragIID, double maxErate, uint32 &numOverlaps) { uint32 tid = omp_get_thread_num(); while (_thread[tid]._batMax <= _cacheLen[fragIID]) { _thread[tid]._batMax *= 2; delete [] _thread[tid]._bat; _thread[tid]._bat = new BAToverlap [_thread[tid]._batMax]; } BAToverlapInt *ptr = _cachePtr[fragIID]; uint32 maxEvalue = AS_OVS_encodeEvalue(maxErate); numOverlaps = 0; for (uint32 pos=0; pos < _cacheLen[fragIID]; pos++) { if (ptr[pos].evalue > maxEvalue) continue; _thread[tid]._bat[numOverlaps].a_hang = ptr[pos].a_hang; _thread[tid]._bat[numOverlaps].b_hang = ptr[pos].b_hang; _thread[tid]._bat[numOverlaps].flipped = ptr[pos].flipped; _thread[tid]._bat[numOverlaps].evalue = ptr[pos].evalue; _thread[tid]._bat[numOverlaps].erate = AS_OVS_decodeEvalue(ptr[pos].evalue); _thread[tid]._bat[numOverlaps].a_iid = fragIID; _thread[tid]._bat[numOverlaps].b_iid = ptr[pos].b_iid; numOverlaps++; } return(_thread[tid]._bat); } void OverlapCache::removeWeakOverlaps(uint32 *minEvalue5p, uint32 *minEvalue3p) { uint32 fiLimit = FI->numFragments(); uint64 saved = 0; uint64 ignored = 0; uint64 removed = 0; for (uint32 fi=1; fi <= fiLimit; fi++) { uint32 numOverlaps = _cacheLen[fi]; BAToverlapInt *ptr = _cachePtr[fi]; for (uint32 pos=0; pos < numOverlaps; pos++) { uint32 aiid = fi; uint32 biid = ptr[pos].b_iid; uint32 evalue = ptr[pos].evalue; // Ignore contained overlaps. if (((ptr[pos].a_hang <= 0) && (ptr[pos].b_hang >= 0)) || ((ptr[pos].a_hang >= 0) && (ptr[pos].b_hang <= 0))) { ignored++; continue; } // Decide which end we need to be looking at. uint32 ta = 0; uint32 tb = 0; if (ptr[pos].a_hang > 0) ta = minEvalue3p[aiid]; else ta = minEvalue5p[aiid]; if (ptr[pos].flipped == false) { if (ptr[pos].b_hang > 0) tb = minEvalue5p[biid]; else tb = minEvalue3p[biid]; } else { if (ptr[pos].b_hang > 0) tb = minEvalue3p[biid]; else tb = minEvalue5p[biid]; } // If the erate is more than the threshold, 'remove' the overlap by maxing out the erate. if ((evalue > ta) || (evalue > tb)) { //fprintf(stdout, "OverlapCache::removeWeakOverlaps()-- remove %7d %7d at %.3f\n", aiid, biid, AS_OVS_decodeEvalue(evalue)); removed++; ptr[pos].evalue = AS_MAX_EVALUE; } else { saved++; } } } writeLog("OverlapCache::removeWeakOverlaps()-- removed "F_U64" weak overlaps.\n", removed); writeLog("OverlapCache::removeWeakOverlaps()-- ignored "F_U64" contained overlaps.\n", ignored); writeLog("OverlapCache::removeWeakOverlaps()-- retained "F_U64" strong overlaps.\n", saved); } #if 0 double OverlapCache::findErate(uint32 aIID, uint32 bIID) { for (uint32 pos=0; pos < _cacheLen[aIID]; pos++) if (_cachePtr[aIID][pos].b_iid == bIID) return(AS_OVS_decodeEvalue(_cachePtr[aIID][pos].evalue)); for (uint32 pos=0; pos < _cacheLen[bIID]; pos++) if (_cachePtr[bIID][pos].b_iid == aIID) return(AS_OVS_decodeEvalue(_cachePtr[bIID][pos].evalue)); return(1.0); } #endif bool OverlapCache::load(const char *prefix, double erate) { char name[FILENAME_MAX]; FILE *file; size_t numRead; sprintf(name, "%s.ovlCache", prefix); if (AS_UTL_fileExists(name, FALSE, FALSE) == false) return(false); fprintf(stderr, "OverlapCache()-- Loading graph from '%s'.\n", name); errno = 0; file = fopen(name, "r"); if (errno) fprintf(stderr, "OverlapCache()-- Failed to open '%s' for reading: %s\n", name, strerror(errno)), exit(1); uint64 magic = ovlCacheMagic; uint32 ovserrbits = AS_MAX_EVALUE_BITS; uint32 ovshngbits = AS_MAX_READLEN_BITS + 1; AS_UTL_safeRead(file, &magic, "overlapCache_magic", sizeof(uint64), 1); AS_UTL_safeRead(file, &ovserrbits, "overlapCache_ovserrbits", sizeof(uint32), 1); AS_UTL_safeRead(file, &ovshngbits, "overlapCache_ovshngbits", sizeof(uint32), 1); if (magic != ovlCacheMagic) fprintf(stderr, "OverlapCache()-- ERROR: File '%s' isn't a bogart ovlCache.\n", name), exit(1); AS_UTL_safeRead(file, &_memLimit, "overlapCache_memLimit", sizeof(uint64), 1); AS_UTL_safeRead(file, &_memUsed, "overlapCache_memUsed", sizeof(uint64), 1); uint32 unused; // Former _batMax, left in for compatibility with old caches. AS_UTL_safeRead(file, &_maxPer, "overlapCache_maxPer", sizeof(uint32), 1); AS_UTL_safeRead(file, &unused, "overlapCache_batMax", sizeof(uint32), 1); _threadMax = omp_get_max_threads(); _thread = new OverlapCacheThreadData [_threadMax]; _cachePtr = new BAToverlapInt * [FI->numFragments() + 1]; _cacheLen = new uint32 [FI->numFragments() + 1]; numRead = AS_UTL_safeRead(file, _cacheLen, "overlapCache_cacheLen", sizeof(uint32), FI->numFragments() + 1); if (numRead != FI->numFragments() + 1) fprintf(stderr, "OverlapCache()-- Short read loading graph '%s'. Fail.\n", name), exit(1); _ovlStoreUniq = NULL; _ovlStoreRept = NULL; fclose(file); // Memory map the overlaps sprintf(name, "%s.ovlCacheDat", prefix); _cacheMMF = new memoryMappedFile(name); _stor = (BAToverlapInt *)_cacheMMF->get(0); // Update pointers into the overlaps _cachePtr[0] = _stor; for (uint32 fi=1; finumFragments() + 1; fi++) _cachePtr[fi] = _cachePtr[fi-1] + _cacheLen[fi-1]; bool doCleaning = false; uint64 nOvl = 0; for (uint32 fi=1; finumFragments() + 1; fi++) { nOvl += _cacheLen[fi]; if ((FI->fragmentLength(fi) == 0) && (_cacheLen[fi] > 0)) doCleaning = true; if (_cacheLen[fi] == 0) _cachePtr[fi] = NULL; } // For each fragment, remove any overlaps to deleted fragments. writeLog("OverlapCache()-- Loaded "F_U64" overlaps.\n", nOvl); if (doCleaning) { uint64 nDel = 0; uint64 nMod = 0; uint64 nOvl = 0; writeLog("OverlapCache()-- Freshly deleted fragments detected. Cleaning overlaps.\n"); char N[FILENAME_MAX]; sprintf(N, "%s.overlapsRemoved.log", prefix); errno = 0; FILE *F = fopen(N, "w"); if (errno) fprintf(stderr, "OverlapCache()-- Failed to open '%s' for writing: %s\n", N, strerror(errno)), exit(1); for (uint32 fi=1; finumFragments() + 1; fi++) { if ((FI->fragmentLength(fi) == 0) && (_cacheLen[fi] > 0)) { nDel++; fprintf(F, "Removing "F_U32" overlaps from deleted deleted fragment "F_U32"\n", _cacheLen[fi], fi); _cachePtr[fi] = NULL; _cacheLen[fi] = 0; } uint32 on = 0; for (uint32 oi=0; oi<_cacheLen[fi]; oi++) { uint32 iid = _cachePtr[fi][oi].b_iid; bool del = (FI->fragmentLength(iid) == 0); if ((del == false) && (on < oi)) _cachePtr[fi][on] = _cachePtr[fi][oi]; if (del == false) on++; } if (_cacheLen[fi] != on) { nMod++; nOvl += _cacheLen[fi] - on; fprintf(F, "Removing "F_U32" overlaps from living fragment "F_U32"\n", _cacheLen[fi] - on, fi); memset(_cachePtr[fi] + on, 0xff, (_cacheLen[fi] - on) * (sizeof(BAToverlapInt))); } _cacheLen[fi] = on; } fclose(F); fprintf(stderr, "OverlapCache()-- Removed all overlaps from "F_U64" deleted fragments. Removed "F_U64" overlaps from "F_U64" alive fragments.\n", nDel, nOvl, nMod); } return(true); } void OverlapCache::save(const char *prefix, double erate) { char name[FILENAME_MAX]; FILE *file; sprintf(name, "%s.ovlCache", prefix); fprintf(stderr, "OverlapCache()-- Saving graph to '%s'.\n", name); errno = 0; file = fopen(name, "w"); if (errno) fprintf(stderr, "OverlapCache()-- Failed to open '%s' for writing: %s\n", name, strerror(errno)), exit(1); uint64 magic = ovlCacheMagic; uint32 ovserrbits = AS_MAX_EVALUE_BITS; uint32 ovshngbits = AS_MAX_READLEN_BITS + 1; AS_UTL_safeWrite(file, &magic, "overlapCache_magic", sizeof(uint64), 1); AS_UTL_safeWrite(file, &ovserrbits, "overlapCache_ovserrbits", sizeof(uint32), 1); AS_UTL_safeWrite(file, &ovshngbits, "overlapCache_ovshngbits", sizeof(uint32), 1); AS_UTL_safeWrite(file, &_memLimit, "overlapCache_memLimit", sizeof(uint64), 1); AS_UTL_safeWrite(file, &_memUsed, "overlapCache_memUsed", sizeof(uint64), 1); AS_UTL_safeWrite(file, &_maxPer, "overlapCache_maxPer", sizeof(uint32), 1); AS_UTL_safeWrite(file, &_maxPer, "overlapCache_batMax", sizeof(uint32), 1); // COMPATIBILITY, REMOVE AS_UTL_safeWrite(file, _cacheLen, "overlapCache_cacheLen", sizeof(uint32), FI->numFragments() + 1); fclose(file); } canu-1.3/src/bogart/AS_BAT_OverlapCache.H000066400000000000000000000161661272335422300201030ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_OverlapCache.H * * Modifications by: * * Brian P. Walenz from 2011-FEB-15 to 2013-AUG-28 * are Copyright 2011-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2011-OCT-13 * are Copyright 2011 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz from 2014-OCT-21 to 2015-JUN-16 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_OVERLAPCACHE #define INCLUDE_AS_BAT_OVERLAPCACHE #include "AS_global.H" #include "ovStore.H" #include "gkStore.H" #include "memoryMappedFile.H" // CA8 used to re-encode the error rate into a smaller-precision number. This was // confusing and broken (it tried to use a log-based encoding to give more precision // to the smaller values). CA3g gives up and uses all 12 bits of precision. // If not enough space for the minimum number of error bits, bump up to a 64-bit word for overlap // storage. // For storing overlaps in memory. 12 bytes per overlap. struct BAToverlapInt { uint64 evalue :AS_MAX_EVALUE_BITS; // 12 by default (same as AS_MAX_EVALUE_BITS) int64 a_hang :AS_MAX_READLEN_BITS+1; // 21+1 by default int64 b_hang :AS_MAX_READLEN_BITS+1; // 21+1 by default uint64 flipped :1; uint32 b_iid; }; #if (AS_MAX_EVALUE_BITS + AS_MAX_READLEN_BITS + 1 + AS_MAX_READLEN_BITS + 1 + 1 > 64) #error not enough bits to store overlaps. decrease AS_MAX_EVALUE_BITS or AS_MAX_READLEN_BITS. #endif // For working with overlaps, 32 bytes per overlap. This data is copied // from the overlap storage (from a BAToverlapInt) with the erate expanded, // and a_iid added. class BAToverlap { public: BAToverlap() { }; ~BAToverlap() { }; public: // Return which end of the read the overlap is on. For 'Ais', the orientation // of B doesn't matter; likewise for 'Bis'. // // If the overlap is a containment relationship, both Is5 and Is3 are false. // bool isDovetail(void) const { return(((a_hang < 0) && (b_hang < 0)) || ((a_hang > 0) && (b_hang > 0))); }; bool AEndIs5prime(void) const { // --------> return((a_hang < 0) && (b_hang < 0)); // ------- }; bool AEndIs3prime(void) const { // --------> return((a_hang > 0) && (b_hang > 0)); // ------- }; bool AisContainer(void) const { // --------> return((a_hang >= 0) && (b_hang <= 0)); // ---- }; bool AisContained(void) const { // ---> return((a_hang <= 0) && (b_hang >= 0)); // --------- }; bool BEndIs3prime(void) const { assert(AisContainer() == false); // Function is not defined assert(AisContained() == false); // for containments. return((AEndIs5prime() && (flipped == false)) || // <=== ------> (AEndIs3prime() && (flipped == true))); // ----> }; bool BEndIs5prime(void) const { assert(AisContainer() == false); // Function is not defined assert(AisContained() == false); // for containments. return((AEndIs5prime() && (flipped == true)) || // ------> (AEndIs3prime() && (flipped == false))); // <=== ----> }; public: int32 a_hang; int32 b_hang; uint32 flipped; uint32 evalue; // Encoded fraction error double erate; // Decoded fraction error uint32 a_iid; uint32 b_iid; }; inline bool BAToverlap_sortByErate(BAToverlap const &a, BAToverlap const &b) { return(a.erate > b.erate); } inline bool BAToverlap_sortByEvalue(BAToverlap const &a, BAToverlap const &b) { return(a.evalue > b.evalue); } class OverlapCacheThreadData { public: OverlapCacheThreadData() { _batMax = 1 * 1024 * 1024; // At 8B each, this is 8MB _bat = new BAToverlap [_batMax]; }; ~OverlapCacheThreadData() { delete [] _bat; }; uint32 _batMax; // For returning overlaps BAToverlap *_bat; // }; class OverlapCache { public: OverlapCache(ovStore *ovlStoreUniq, ovStore *ovlStoreRept, const char *prefix, double maxErate, uint32 minOverlap, uint64 maxMemory, uint32 maxOverlaps, bool onlysave, bool dosave); ~OverlapCache(); void computeOverlapLimit(void); uint32 filterOverlaps(uint32 maxOVSerate, uint32 minOverlap, uint32 no); void loadOverlaps(double erate, uint32 minOverlap, const char *prefix, bool onlySave, bool doSave); BAToverlap *getOverlaps(uint32 fragIID, double maxErate, uint32 &numOverlaps); void removeWeakOverlaps(uint32 *minEvalue5p, uint32 *minEvalue3p); //double findErate(uint32 aIID, uint32 bIID); private: bool load(const char *prefix, double erate); void save(const char *prefix, double erate); private: uint64 _memLimit; uint64 _memUsed; uint32 _storMax; // Size of the heap uint32 _storLen; // Position we are at in this heap BAToverlapInt *_stor; // Pointer to the heap (probably the last thing on _heaps) vector _heaps; memoryMappedFile *_cacheMMF; BAToverlapInt **_cachePtr; // Mapping of frag iid to overlaps stored in the heap uint32 *_cacheLen; // Number of overlaps per frag iid uint32 _maxPer; // Maximum number of overlaps to load for a single fragment uint32 _ovsMax; // For loading overlaps ovOverlap *_ovs; // uint64 *_ovsSco; // For scoring overlaps during the load uint64 *_ovsTmp; // For picking out a score threshold uint64 _threadMax; OverlapCacheThreadData *_thread; ovStore *_ovlStoreUniq; // Pointers to input stores ovStore *_ovlStoreRept; }; extern OverlapCache *OC; #endif // INCLUDE_AS_BAT_OVERLAPCACHE canu-1.3/src/bogart/AS_BAT_PlaceContains.C000066400000000000000000000210131272335422300202500ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_PlaceContains.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2014-JAN-29 * are Copyright 2010-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2015-JUN-03 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_PlaceContains.H" #include "AS_BAT_PlaceFragUsingOverlaps.H" #define SHOW_PLACEMENT_DETAIL // Reports evidence (too much) for placing reads. #define SHOW_PLACEMENT // Reports where the read was placed. void breakSingletonTigs(UnitigVector &unitigs) { // For any singleton unitig, eject the read and delete the unitig. Eventually, // we will stop making singleton unitigs. uint32 removed = 0; for (uint32 ti=1; tiufpath.size() > 1) continue; unitigs[ti] = NULL; // Remove the unitig from the list utg->removeFrag(utg->ufpath[0].ident); // Eject the read delete utg; // Reclaim space removed++; // Count } writeLog("Removed %u read%s from %u singleton unitig%s.\n", removed, (removed != 1) ? "" : "s", removed, (removed != 1) ? "" : "s"); } void placeUnplacedUsingAllOverlaps(UnitigVector &unitigs, const char *prefix) { uint32 fiLimit = FI->numFragments(); uint32 numThreads = omp_get_max_threads(); uint32 blockSize = (fiLimit < 100 * numThreads) ? numThreads : fiLimit / 99; uint32 *placedTig = new uint32 [FI->numFragments() + 1]; SeqInterval *placedPos = new SeqInterval [FI->numFragments() + 1]; memset(placedTig, 0, sizeof(uint32) * (FI->numFragments() + 1)); memset(placedPos, 0, sizeof(SeqInterval) * (FI->numFragments() + 1)); // Just some logging. Count the number of reads we try to place. uint32 nToPlaceContained = 0; uint32 nToPlace = 0; uint32 nPlacedContained = 0; uint32 nPlaced = 0; uint32 nFailedContained = 0; uint32 nFailed = 0; for (uint32 fid=1; fidnumFragments()+1; fid++) if (Unitig::fragIn(fid) == 0) if (OG->isContained(fid)) nToPlaceContained++; else nToPlace++; writeLog("placeContains()-- placing %u contained and %u unplaced reads, with %d threads.\n", nToPlaceContained, nToPlace, numThreads); // Do the placing! #pragma omp parallel for schedule(dynamic, blockSize) for (uint32 fid=1; fidnumFragments()+1; fid++) { bool enableLog = true; if (Unitig::fragIn(fid) > 0) continue; // Place the read. vector placements; placeFragUsingOverlaps(unitigs, AS_MAX_ERATE, NULL, fid, placements); // Search the placements for the highest expected identity placement using all overlaps in the unitig. uint32 b = UINT32_MAX; for (uint32 i=0; iufpath.size() == 1) // Ignore placements in singletons. continue; uint32 bgn = (placements[i].position.bgn < placements[i].position.end) ? placements[i].position.bgn : placements[i].position.end; uint32 end = (placements[i].position.bgn < placements[i].position.end) ? placements[i].position.end : placements[i].position.bgn; double erate = placements[i].errors / placements[i].aligned; if (tig->overlapConsistentWithTig(5.0, bgn, end, erate) < 0.5) { if ((enableLog == true) && (logFileFlagSet(LOG_PLACE_UNPLACED))) writeLog("frag %8u tested tig %6u (%6u reads) at %8u-%8u (cov %7.5f erate %6.4f) - HIGH ERROR\n", fid, placements[i].tigID, tig->ufpath.size(), placements[i].position.bgn, placements[i].position.end, placements[i].fCoverage, erate); continue; } if ((enableLog == true) && (logFileFlagSet(LOG_PLACE_UNPLACED))) writeLog("frag %8u tested tig %6u (%6u reads) at %8u-%8u (cov %7.5f erate %6.4f)\n", fid, placements[i].tigID, tig->ufpath.size(), placements[i].position.bgn, placements[i].position.end, placements[i].fCoverage, erate); if ((b == UINT32_MAX) || (placements[i].errors / placements[i].aligned < placements[b].errors / placements[b].aligned)) b = i; } // If we didn't find a best, b will be invalid; set positions for adding to a new tig. // If we did, save both the position it was placed at, and the tigID it was placed in. if (b == UINT32_MAX) { if ((enableLog == true) && (logFileFlagSet(LOG_PLACE_UNPLACED))) writeLog("frag %8u remains unplaced\n", fid); placedPos[fid].bgn = 0; placedPos[fid].end = FI->fragmentLength(fid); } else { if ((enableLog == true) && (logFileFlagSet(LOG_PLACE_UNPLACED))) writeLog("frag %8u placed tig %6u (%6u reads) at %8u-%8u (cov %7.5f erate %6.4f)\n", fid, placements[b].tigID, unitigs[placements[b].tigID]->ufpath.size(), placements[b].position.bgn, placements[b].position.end, placements[b].fCoverage, placements[b].errors / placements[b].aligned); placedTig[fid] = placements[b].tigID; placedPos[fid] = placements[b].position; } } // All reads placed, now just dump them in their correct tigs. for (uint32 fid=1; fidnumFragments()+1; fid++) { Unitig *tig = NULL; ufNode frg; if (Unitig::fragIn(fid) > 0) continue; // If not placed, dump it in a new unitig. Well, not anymore. These reads were not placed in // any tig initially, were not allowed to seed a tig, and now, could find no place to go. // They're garbage. Plus, it screws up the logging above because we don't know the new tig ID // until now. if (placedTig[fid] == 0) { if (OG->isContained(fid)) nFailedContained++; else nFailed++; //tig = unitigs.newUnitig(false); } // Otherwise, it was placed somewhere, grab the tig. else { if (OG->isContained(fid)) nPlacedContained++; else nPlaced++; tig = unitigs[placedTig[fid]]; } // Regardless, add it to the tig. Logging for this is above. if (tig) { frg.ident = fid; frg.contained = 0; frg.parent = 0; frg.ahang = 0; frg.bhang = 0; frg.position = placedPos[fid]; tig->addFrag(frg, 0, false); } } // Cleanup. delete [] placedPos; delete [] placedTig; writeLog("placeContains()-- Placed %u contained reads and %u unplaced reads.\n", nPlacedContained, nPlaced); writeLog("placeContains()-- Failed to place %u contained reads (too high error suspected) and %u unplaced reads (lack of overlaps suspected).\n", nFailedContained, nFailed); // But wait! All the tigs need to be sorted. Well, not really _all_, but the hard ones to sort // are big, and those quite likely had reads added to them, so it's really not worth the effort // of tracking which ones need sorting, since the ones that don't need it are trivial to sort. for (uint32 ti=1; tisort(); } } canu-1.3/src/bogart/AS_BAT_PlaceContains.H000066400000000000000000000031661272335422300202660ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_PlaceContains.H * * Modifications by: * * Brian P. Walenz from 2010-DEC-06 to 2013-AUG-01 * are Copyright 2010-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2015-JUN-03 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_PLACECONTAINS #define INCLUDE_AS_BAT_PLACECONTAINS #include "AS_BAT_UnitigVector.H" void breakSingletonTigs(UnitigVector &unitigs); void placeUnplacedUsingAllOverlaps(UnitigVector &unitigs, const char *prefix); #endif // INCLUDE_AS_BAT_PLACECONTAINS canu-1.3/src/bogart/AS_BAT_PlaceFragUsingOverlaps.C000066400000000000000000000562301272335422300221040ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_PlaceFragUsingOverlaps.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-SEP-08 * are Copyright 2010-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-09 to 2015-AUG-05 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_PlaceFragUsingOverlaps.H" #include "intervalList.H" // Report LOTS of details on placement, including evidence. #undef VERBOSE_PLACEMENT bool placeFragUsingOverlaps(UnitigVector &unitigs, double erate, Unitig *target, uint32 fid, vector &placements) { //logFileFlags |= LOG_PLACE_FRAG; if (logFileFlagSet(LOG_PLACE_FRAG)) writeLog("placeFragUsingOverlaps()-- begin for frag %d into target tig %d\n", fid, target->id()); assert(fid > 0); assert(fid <= FI->numFragments()); ufNode frag; frag.ident = fid; frag.contained = 0; frag.parent = 0; frag.ahang = 0; frag.bhang = 0; frag.position.bgn = 0; frag.position.end = 0; placements.clear(); uint32 ovlLen = 0; BAToverlap *ovl = OC->getOverlaps(frag.ident, erate, ovlLen); overlapPlacement *ovlPlace = new overlapPlacement[ovlLen]; uint32 nFragmentsNotPlaced = 0; // Initialize placements to nowhere. for (uint32 i=0; iplaceFrag(frag, fid, ovl[i].AEndIs3prime(), &edge) == false) { nFragmentsNotPlaced++; continue; } uint32 olen = FI->overlapLength(ovl[i].a_iid, ovl[i].b_iid, ovl[i].a_hang, ovl[i].b_hang); uint32 flen = FI->fragmentLength(ovl[i].a_iid); ovlPlace[i].frgID = frag.ident; ovlPlace[i].refID = ovl[i].b_iid; ovlPlace[i].tigID = tig->id(); ovlPlace[i].position = frag.position; ovlPlace[i].errors = olen * ovl[i].erate; ovlPlace[i].covered.bgn = (ovl[i].a_hang < 0) ? 0 : ovl[i].a_hang; ovlPlace[i].covered.end = (ovl[i].b_hang > 0) ? flen : ovl[i].b_hang + flen; ovlPlace[i].aligned = ovlPlace[i].covered.end - ovlPlace[i].covered.bgn; assert(ovlPlace[i].covered.bgn < ovlPlace[i].covered.end); // Compute the portion of the unitig that is actually verified by // the overlap. if (ovlPlace[i].position.bgn < ovlPlace[i].position.end) { ovlPlace[i].verified.bgn = ovlPlace[i].position.bgn + ovlPlace[i].covered.bgn; ovlPlace[i].verified.end = ovlPlace[i].position.bgn + ovlPlace[i].covered.end; if (ovlPlace[i].verified.end > ovlPlace[i].position.end) ovlPlace[i].verified.end = ovlPlace[i].position.end; assert(ovlPlace[i].verified.bgn >= ovlPlace[i].position.bgn); assert(ovlPlace[i].verified.end <= ovlPlace[i].position.end); assert(ovlPlace[i].verified.bgn < ovlPlace[i].verified.end); } else { ovlPlace[i].verified.bgn = ovlPlace[i].position.bgn - ovlPlace[i].covered.bgn; // pos.bgn is the larger and cov.bgn the smaller, so ver.bgn is the larger ovlPlace[i].verified.end = ovlPlace[i].position.bgn - ovlPlace[i].covered.end; // pos.bgn is the larger and cov.bgn the larger, so ver.end the smaller if (ovlPlace[i].verified.end < ovlPlace[i].position.end) ovlPlace[i].verified.end = ovlPlace[i].position.end; assert(ovlPlace[i].verified.end >= ovlPlace[i].position.end); assert(ovlPlace[i].verified.bgn <= ovlPlace[i].position.bgn); assert(ovlPlace[i].verified.end < ovlPlace[i].verified.bgn); } // Disallow any placements that exceed the boundary of the unitig. These cannot be confirmed // by overlaps and might be wrong. Sample cases: // o sticking a unique/repeat fragment onto a repeat (leaving the unique uncovered) // o sticking a chimeric fragment onto the end of a unitig (leaving the chimeric join uncovered) if ((MIN(ovlPlace[i].position.bgn, ovlPlace[i].position.end) < 0) || (MAX(ovlPlace[i].position.bgn, ovlPlace[i].position.end) > tig->getLength())) { #ifdef VERBOSE_PLACEMENT //if (logFileFlagSet(LOG_PLACE_FRAG)) writeLog("placeFragUsingOverlaps()-- frag %d in unitig %d at %d,%d (verified %d,%d) from overlap ident %d %d hang %d %d flipped %d covered %d,%d DISALLOWED\n", frag.ident, tig->id(), ovlPlace[i].position.bgn, ovlPlace[i].position.end, ovlPlace[i].verified.bgn, ovlPlace[i].verified.end, ovl[i].a_iid, ovl[i].b_iid, ovl[i].a_hang, ovl[i].b_hang, ovl[i].flipped, ovlPlace[i].covered.bgn, ovlPlace[i].covered.end); #endif ovlPlace[i] = overlapPlacement(); } else { #ifdef VERBOSE_PLACEMENT //if (logFileFlagSet(LOG_PLACE_FRAG)) writeLog("placeFragUsingOverlaps()-- frag %d in unitig %d at %d,%d (verified %d,%d) from overlap ident %d %d hang %d %d flipped %d covered %d,%d\n", frag.ident, tig->id(), ovlPlace[i].position.bgn, ovlPlace[i].position.end, ovlPlace[i].verified.bgn, ovlPlace[i].verified.end, ovl[i].a_iid, ovl[i].b_iid, ovl[i].a_hang, ovl[i].b_hang, ovl[i].flipped, ovlPlace[i].covered.bgn, ovlPlace[i].covered.end); #endif } assert((ovlPlace[i].position.bgn < ovlPlace[i].position.end) == (ovlPlace[i].verified.bgn < ovlPlace[i].verified.end)); } // Over all overlaps. // Report if any of the placement routines fail. This shouldn't happen, but if it does, it is // hardly fatal. #ifdef VERBOSE_PLACEMENT if (nFragmentsNotPlaced > 0) //if (logFileFlagSet(LOG_PLACE_FRAG)) writeLog("placeFragUsingOverlaps()-- WARNING: Failed to place %d fragments\n", nFragmentsNotPlaced); #endif // Sort all the placements. Any overlap we couldn't place is automatically in Unitig 0, the // invalid unitig. Sort order is by unitig ID, then by orientation, then by position. // sort(ovlPlace, ovlPlace + ovlLen, overlapPlacement_byLocation); // Segregate the overlaps by placement in the unitig. We want to construct one // overlapPlacement for each distinct placement. How this is done: // // For all overlapping overlaps for a specific unitig and a specific fragment orientation, the // end points (+- a few bases) are added to an interval list. The intervals are combined. // Each combined interval now forms the basis of a cluster of overlaps. A list of the pairs of // clusters hit by each overlap is built. If there is one clear winner, that is picked. If // there is no clear winner, the fragment cannot be placed. // // unitig ========================================================================== // overlaps -------------------- ------------------ // ------------------- ------------------- // ------------------- ------------------- // intervals x1x x2x x3x x4x x5x x6xx // // This unitig has two sets of "overlapping overlaps". The left set could be from a tandem // repeat. We'll get two overlaps to the 2,4 pair, and one overlap to the 1,3 pair. Assuming // that is good enough to be a clear winner, we'll ignore the 1,3 overlap and compute position // based on the other two overlaps. uint32 bgn = 0; // Range of overlaps with the same unitig/orientation uint32 end = 1; // Skip overlaps that didn't generate a placement while ((bgn < ovlLen) && (ovlPlace[bgn].tigID == 0)) bgn++; // Process all placements. while (bgn < ovlLen) { // Find the last placement with the same unitig/orientation as the 'bgn' fragment. // Orientation of 'position' and 'verified' is the same, asserted above. end = bgn + 1; while ((end < ovlLen) && (ovlPlace[bgn].tigID == ovlPlace[end].tigID) && (ovlPlace[bgn].verified.isReverse() == ovlPlace[end].verified.isReverse())) end++; // Over all placements with the same unitig/orientation (that'd be from bgn to end), build // interval lists for the begin point and the end point. Remember, this is all fragments to a // single unitig (the whole picture above), not just the overlapping fragment sets (left or // right blocks). // // This used to (before MAY-2016) use the 'verified' placement, instead of the 'full' placement. // In long pacbio reads, this seems to result in far too many clusters - each placement is // derived from one overlap, which will almost never cover the whole read. intervalList bgnPoints; intervalList endPoints; int32 windowSlop = 0.075 * FI->fragmentLength(frag.ident); if (windowSlop < 5) windowSlop = 5; #ifdef VERBOSE_PLACEMENT writeLog("placeFragUsingOverlaps()-- windowSlop = %d\n", windowSlop); #endif for (uint32 oo=bgn; oo 0); int32 bb = ovlPlace[oo].position.bgn; int32 be = ovlPlace[oo].position.bgn + windowSlop; int32 eb = ovlPlace[oo].position.end; int32 ee = ovlPlace[oo].position.end + windowSlop; bb = (bb < windowSlop) ? 0 : bb - windowSlop; eb = (eb < windowSlop) ? 0 : eb - windowSlop; bgnPoints.add(bb, be - bb); endPoints.add(eb, ee - eb); } bgnPoints.merge(); endPoints.merge(); // Now, assign each placement to a end-pair cluster based on the interval ID that the end point // falls in. // // count the number of fragments that hit each pair of points. We can do this two ways: // 1) With a list of point-pairs that we sort and count -- O(n) size, O(n log n) time // 2) With an array of all point-pairs that we increment directly -- O(p*p) size, O(n) time // Typically, p is small. int32 numBgnPoints = bgnPoints.numberOfIntervals(); int32 numEndPoints = endPoints.numberOfIntervals(); for (uint32 oo=bgn; oofragmentLength(op.frgID); // Find the first and last fragment in the unitig that we overlap with. // // The first fragment is easy. We can run through the list of overlaps, ask the unitig for // the ordinal of each fragment, and return the lowest. // // The last fragment is similar, but we need to return the highest ordinal that is also the // longest. (In the first fragment case, we are guaranteed by the construction of the unitig // to have the earliest fragment position). Unitig *destTig = unitigs[op.tigID]; uint32 firstOrdinal = UINT32_MAX; uint32 firstPosition = UINT32_MAX; uint32 lastOrdinal = 0; uint32 lastPosition = 0; FragmentEnd firstEnd; FragmentEnd lastEnd; for (uint32 oo=os; oopathPosition(ovlPlace[oo].refID); ufNode &ovlFrg = destTig->ufpath[ordinal]; uint32 minPos = MIN(ovlFrg.position.bgn, ovlFrg.position.end); uint32 maxPos = MAX(ovlFrg.position.bgn, ovlFrg.position.end); //writeLog("placeFragUsingOverlaps()-- PickEnds ordinal %d tigFrg %d pos %d,%d\n", // ordinal, ovlFrg.ident, minPos, maxPos); // For a normal dovetail alignment, this is pretty straight forward. We pick the // end we align to. // // For spur alignments (repeat detection) it is backwards to what we want. More comments // in repeatJunctionEvidence::repeatJunctionEvidence(). // // \ / // -----alignedfragment---- // ------.... .....----- // if (((minPos < firstPosition)) || ((minPos <= firstPosition) && (ordinal < firstOrdinal))) { firstOrdinal = ordinal; firstPosition = minPos; firstEnd = FragmentEnd(ovlFrg.ident, (ovlFrg.position.bgn < ovlFrg.position.end)); } if (((maxPos > lastPosition)) || ((maxPos >= lastPosition) && (ordinal > lastOrdinal))) { lastOrdinal = ordinal; lastPosition = minPos; lastEnd = FragmentEnd(ovlFrg.ident, (ovlFrg.position.end < ovlFrg.position.bgn)); } } if (nForward > 0) { op.frag5p = firstEnd; op.frag3p = lastEnd; } else { op.frag5p = lastEnd; op.frag3p = firstEnd; } // Compute mean and stddev placement. uint32 numPlace = 0; double bgnMean = 0; double endMean = 0; op.bgnStdDev = 0.0; op.endStdDev = 0.0; for (uint32 oo=os; oo= ovlPlace[oo].verified.end) writeLog("placeFragUsingOverlaps()-- frag %d FWD verified placement invalid (bgn,end %d,%d) for position (bgn,end %d,%d)\n", ovlPlace[oo].frgID, ovlPlace[oo].verified.bgn, ovlPlace[oo].verified.end, ovlPlace[oo].position.bgn, ovlPlace[oo].position.end); assert(ovlPlace[oo].verified.bgn < ovlPlace[oo].verified.end); bgnMean += ovlPlace[oo].position.bgn; endMean += ovlPlace[oo].position.end; op.verified.bgn = MIN(op.verified.bgn, ovlPlace[oo].verified.bgn); op.verified.end = MAX(op.verified.end, ovlPlace[oo].verified.end); } else { if (ovlPlace[oo].verified.bgn < ovlPlace[oo].verified.end) writeLog("placeFragUsingOverlaps()-- frag %d REV verified placement invalid (bgn,end %d,%d) for position (bgn,end %d,%d)\n", ovlPlace[oo].frgID, ovlPlace[oo].verified.bgn, ovlPlace[oo].verified.end, ovlPlace[oo].position.bgn, ovlPlace[oo].position.end); assert(ovlPlace[oo].verified.bgn >= ovlPlace[oo].verified.end); bgnMean += ovlPlace[oo].position.end; endMean += ovlPlace[oo].position.bgn; op.verified.bgn = MAX(op.verified.bgn, ovlPlace[oo].verified.bgn); op.verified.end = MIN(op.verified.end, ovlPlace[oo].verified.end); } numPlace++; } bgnMean /= numPlace; endMean /= numPlace; op.position.bgn = (int32)((nReverse == 0) ? bgnMean : endMean); op.position.end = (int32)((nReverse == 0) ? endMean : bgnMean); for (uint32 oo=os; oofragmentLength(op.frgID)); if ((op.bgnStdDev > allowableStdDev) || (op.endStdDev > allowableStdDev)) weakStdDev = true; if (((op.position.bgn < op.position.end) && (op.position.bgn + 3 * op.bgnStdDev > op.position.end - 3 * op.endStdDev)) || ((op.position.end < op.position.bgn) && (op.position.end + 3 * op.endStdDev > op.position.bgn - 3 * op.bgnStdDev))) overlappingSpan = true; int32 poslen = (op.position.end > op.position.bgn) ? (op.position.end - op.position.bgn) : (op.position.bgn - op.position.end); int32 trulen = FI->fragmentLength(op.frgID); double scaled = (double)poslen / trulen; if ((scaled < 0.3333) || (2.0 < scaled)) spanBad = true; if ((weakStdDev) && (0)) // Read is not known to have lots of indel, but the stddev is high. reject = true; if (overlappingSpan) // Read placements are conflicting and overlapping. reject = true; if (spanBad) // Bogus placement, more than twice as large as expected, or less than 1/3 expected. reject = true; if (reject) { #ifdef VERBOSE_PLACEMENT //if (logFileFlagSet(LOG_PLACE_FRAG)) { writeLog("placeFragUsingOverlaps()-- frag %d in unitig %d at %d,%d (+- %.2f,%.2f) -- cov %.2f (%d,%d) errors %.2f aligned %d novl %d -- INVALID stddev weak %d overlapping %d bad size %d\n", op.frgID, op.tigID, op.position.bgn, op.position.end, op.bgnStdDev, op.endStdDev, op.fCoverage, op.covered.bgn, op.covered.end, op.errors, op.aligned, oe - os, weakStdDev, overlappingSpan, spanBad); for (uint32 oo=os; oo &placements); #endif // INCLUDE_AS_BAT_PLACEFRAGUSINGOVERLAPS canu-1.3/src/bogart/AS_BAT_PopBubbles.C000066400000000000000000000606351272335422300175770ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAR-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_OverlapCache.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_PlaceFragUsingOverlaps.H" #include "AS_BAT_Instrumentation.H" #include "intervalList.H" #include #include #include using namespace std; #define BUBBLE_READ_FRACTION 0.5 #undef SHOW_MULTIPLE_PLACEMENTS // Reports reads that are placed multiple times in a single target region class candidatePop { public: candidatePop() { }; candidatePop(Unitig *bubble_, Unitig *target_, uint32 bgn_, uint32 end_) { bubble = bubble_; target = target_; bgn = bgn_; end = end_; }; ~candidatePop() { }; Unitig *bubble; Unitig *target; uint32 bgn; uint32 end; vector placed; }; // A list of the target unitigs that a bubble could be popped into. typedef map > BubTargetList; // Decide which unitigs can be bubbles. The first pass finds unitigs that can be potential // bubbles. Any unitig where every dovetail read has an overlap to some other unitig is a // candidate for bubble popping. void findPotentialBubbles(UnitigVector &unitigs, BubTargetList &potentialBubbles) { uint32 tiLimit = unitigs.size(); uint32 tiNumThreads = omp_get_max_threads(); uint32 tiBlockSize = (tiLimit < 100000 * tiNumThreads) ? tiNumThreads : tiLimit / 99999; writeLog("bubbleDetect()-- working on "F_U32" unitigs, with "F_U32" threads.\n", tiLimit, tiNumThreads); for (uint32 ti=0; tiufpath.size() == 1)) // Singleton, handled elsewhere. continue; uint32 nonContainedReads = 0; bool validBubble = true; map tigOlapsTo; uint32 fiLimit = tig->ufpath.size(); uint32 fiNumThreads = omp_get_max_threads(); uint32 fiBlockSize = (fiLimit < 100 * fiNumThreads) ? fiNumThreads : fiLimit / 99; for (uint32 fi=0; (validBubble == true) && (fiufpath[fi].ident; if (OG->isContained(rid) == true) // Don't need to check contained reads. If their container continue; // passes the tests below, the contained read will too. nonContainedReads++; uint32 ovlLen = 0; BAToverlap *ovl = OC->getOverlaps(rid, AS_MAX_ERATE, ovlLen); set readOlapsTo; for (uint32 oi=0; oiufpath.size() == 1) || (ovlTig->id() == tig->id()) || (ovlTig->getLength() < tig->getLength())) continue; // Otherwise, remember that we had an overlap to ovlTig. //writeLog("tig %u read %u overlap to tig %u read %u\n", // tig->id(), rid, ovlTigID, ovl[oi].b_iid); readOlapsTo.insert(ovlTigID); } //writeLog("tig %8u read %8u has %u olaps\n", tig->id(), rid, readOlapsTo.size()); // Transfer the per-read counts to the per-unitig counts: add one to the counter for each tig // that we have overlaps to. for (set::iterator it=readOlapsTo.begin(); it != readOlapsTo.end(); ++it) tigOlapsTo[*it]++; // Decide if we're a valid potential bubble. If tig id (in it->first) has overlaps to every // read we've seen so far (nonContainedReads), we're still a valid bubble. // // To _attempt_ to have differences in the bubble, we'll accept it if 3/4 of the reads // have overlaps. validBubble = false; for (map::iterator it=tigOlapsTo.begin(); it != tigOlapsTo.end(); ++it) if (it->second >= BUBBLE_READ_FRACTION * nonContainedReads) validBubble = true; // If we've not seen that many reads, pretend it's a valid bubble. It'll get screened out later. if (nonContainedReads < 16) validBubble = true; } // If not validBubble, report. #if 0 if (validBubble == false) { writeLog("notValidBubble tig %8d expects %6u reads\n", tig->id(), nonContainedReads); for (map::iterator it=tigOlapsTo.begin(); it != tigOlapsTo.end(); ++it) writeLog(" to tig %8u overlaps %6u\n", it->first, it->second); } #endif // If validBubble, then there is a tig that every dovetail read has at least one overlap to. // Save those tigs in potentialBubbles. uint32 nTigs = 0; if (validBubble) { for (map::iterator it=tigOlapsTo.begin(); it != tigOlapsTo.end(); ++it) if (it->second >= BUBBLE_READ_FRACTION * nonContainedReads) nTigs++; } // ALWAYS log potential bubbles. if (nTigs > 0) { writeLog("\n"); writeLog("potential bubble tig %8u length %9u nReads %7u to %3u tigs:\n", tig->id(), tig->getLength(), tig->ufpath.size(), nTigs); for (map::iterator it=tigOlapsTo.begin(); it != tigOlapsTo.end(); ++it) { if (it->second >= BUBBLE_READ_FRACTION * nonContainedReads) { Unitig *dest = unitigs[it->first]; writeLog(" tig %8u length %9u nReads %7u\n", dest->id(), dest->getLength(), dest->ufpath.size()); potentialBubbles[ti].push_back(dest->id()); } } } } flushLog(); } // Find filtered placements for all the reads in the potential bubble tigs. vector * findBubbleReadPlacements(UnitigVector &unitigs, BubTargetList &potentialBubbles, double deviationBubble) { uint32 fiLimit = FI->numFragments(); uint32 fiNumThreads = omp_get_max_threads(); uint32 fiBlockSize = (fiLimit < 1000 * fiNumThreads) ? fiNumThreads : fiLimit / 999; vector *placed = new vector [fiLimit + 1]; #pragma omp parallel for schedule(dynamic, fiBlockSize) for (uint32 fi=0; fiisContained(fi)) || // Read is contained, ignore it. (potentialBubbles.count(rdAtigID) == 0)) // Read isn't in a potential bubble, ignore it. continue; Unitig *rdAtig = unitigs[rdAtigID]; ufNode *rdA = &rdAtig->ufpath[ Unitig::pathPosition(fi) ]; bool rdAfwd = (rdA->position.bgn < rdA->position.end); int32 rdAlo = (rdAfwd) ? rdA->position.bgn : rdA->position.end; int32 rdAhi = (rdAfwd) ? rdA->position.end : rdA->position.bgn; uint32 ovlLen = 0; BAToverlap *ovl = OC->getOverlaps(rdA->ident, AS_MAX_ERATE, ovlLen); set intersections; //if ((fi % 100) == 0) // fprintf(stderr, "findBubbleReadPlacements()-- read %8u with %6u overlaps - %6.2f%% finished.\r", // rdA->ident, ovlLen, 100.0 * fi / fiLimit); // Compute all placements for this read. vector placements; placeFragUsingOverlaps(unitigs, AS_MAX_ERATE, NULL, rdA->ident, placements); // Weed out placements that aren't for bubbles, or that are for bubbles but are poor quality. Or are to ourself! for (uint32 pi=0; pi tig %6u %6u reads at %8u-%8u (cov %7.5f erate %6.4f) - SAME TIG\n", // rdAtigID, placements[pi].frgID, placements[pi].tigID, rdBtig->ufpath.size(), placements[pi].position.bgn, placements[pi].position.end, placements[pi].fCoverage, erate); continue; } // Ignore the placement if it is to a non-tig / singleton read, or if it didn't place the // read fully. if ((rdBtigID == 0) || (rdBtig == NULL) || (rdBtig->ufpath.size() == 1) || (placements[pi].fCoverage < 0.99)) { if (logFileFlagSet(LOG_BUBBLE_DETAIL)) writeLog("tig %6u frag %8u -> tig %6u %6u reads at %8u-%8u (cov %7.5f erate %6.4f) - PARTIALLY PLACED\n", rdAtigID, placements[pi].frgID, placements[pi].tigID, rdBtig->ufpath.size(), placements[pi].position.bgn, placements[pi].position.end, placements[pi].fCoverage, erate); continue; } // Ignore the placement if it isn't to one of our bubble-popping candidate unitigs. bool dontcare = true; vector &pbubbles = potentialBubbles[rdAtigID]; for (uint32 pb=0; pb tig %6u %6u reads at %8u-%8u (cov %7.5f erate %6.4f) - NOT CANDIDATE TIG\n", rdAtigID, placements[pi].frgID, placements[pi].tigID, rdBtig->ufpath.size(), placements[pi].position.bgn, placements[pi].position.end, placements[pi].fCoverage, erate); continue; } // Ignore the placement if it is too diverged from the destination tig. if (rdBtig->overlapConsistentWithTig(deviationBubble, lo, hi, erate) < 0.5) { if (logFileFlagSet(LOG_BUBBLE_DETAIL)) writeLog("tig %6u frag %8u -> tig %6u %6u reads at %8u-%8u (cov %7.5f erate %6.4f) - HIGH ERROR\n", rdAtigID, placements[pi].frgID, placements[pi].tigID, rdBtig->ufpath.size(), placements[pi].position.bgn, placements[pi].position.end, placements[pi].fCoverage, erate); continue; } // Good placement! if (logFileFlagSet(LOG_BUBBLE_DETAIL)) writeLog("tig %6u frag %8u -> tig %6u %6u reads at %8u-%8u (cov %7.5f erate %6.4f)\n", rdAtigID, placements[pi].frgID, placements[pi].tigID, rdBtig->ufpath.size(), placements[pi].position.bgn, placements[pi].position.end, placements[pi].fCoverage, erate); placed[fi].push_back(placements[pi]); } } return(placed); } // Bubble popping cannot be done in parallel -- there is a race condition when both unitigs // A and B are considering merging in unitig C. void popBubbles(UnitigVector &unitigs, double deviationBubble) { BubTargetList potentialBubbles; findPotentialBubbles(unitigs, potentialBubbles); writeLog("\n"); writeLog("Found "F_SIZE_T" potential bubbles.\n", potentialBubbles.size()); writeLog("\n"); vector *placed = findBubbleReadPlacements(unitigs, potentialBubbles, deviationBubble); // We now have, in 'placed', a list of all the places that each read could be placed. Decide if there is a _single_ // place for each bubble to be popped. uint32 tiLimit = unitigs.size(); //uint32 tiNumThreads = omp_get_max_threads(); //uint32 tiBlockSize = (tiLimit < 100000 * tiNumThreads) ? tiNumThreads : tiLimit / 99999; // Clear flags. for (uint32 ti=0; ti_isBubble = false; unitigs[ti]->_isRepeat = false; } } // In parallel, process the placements. for (uint32 ti=0; tiufpath.size(); fi++) { uint32 readID = bubble->ufpath[fi].ident; if (placed[readID].size() > 0) hasPlacements = true; } if (hasPlacements == false) writeLog("potential bubble %u had no valid placements (all were not contained in target tig)\n", ti); else writeLog("potential bubble %u\n", ti); // Split the placements into piles for each target and build an interval list for each target. // For each read in the tig, convert the vector of placements into interval lists, one list per target tig. map *> targetIntervals; for (uint32 fi=0; fiufpath.size(); fi++) { uint32 readID = bubble->ufpath[fi].ident; for (uint32 pp=0; pp 0); uint32 bgn = (placed[readID][pp].position.bgn < placed[readID][pp].position.end) ? placed[readID][pp].position.bgn : placed[readID][pp].position.end; uint32 end = (placed[readID][pp].position.bgn < placed[readID][pp].position.end) ? placed[readID][pp].position.end : placed[readID][pp].position.bgn; if (targetIntervals[tid] == NULL) targetIntervals[tid] = new intervalList; //writeLog("read %u -> tig %u intervals %u-%u\n", readID, tid, bgn, end); targetIntervals[tid]->add(bgn, end-bgn); } } vector targets; // Squish the intervals. Create new candidatePops for each interval that isn't too big or // small. Assign each overlapPlacements to the correct candidatePop. for (map *>::iterator it=targetIntervals.begin(); it != targetIntervals.end(); ++it) { uint32 targetID = it->first; intervalList *IL = it->second; IL->merge(); // Discard intervals that are significantly too small or large. Save the ones that are // nicely sized. Logging here isn't terribly useful, it's just repeated (out of order) later // when we try to make sense of the read alignments. for (uint32 ii=0; iinumberOfIntervals(); ii++) { if ((IL->hi(ii) - IL->lo(ii) < 0.75 * bubble->getLength()) || // Too small! (1.25 * bubble->getLength() < IL->hi(ii) - IL->lo(ii))) { // Too big! writeLog("tig %8u length %9u -> target %8u piece %2u position %9u-%9u length %8u - size mismatch, discarded\n", bubble->id(), bubble->getLength(), targetID, ii, IL->lo(ii), IL->hi(ii), IL->hi(ii) - IL->lo(ii)); continue; } writeLog("tig %8u length %9u -> target %8u piece %2u position %9u-%9u length %8u\n", bubble->id(), bubble->getLength(), targetID, ii, IL->lo(ii), IL->hi(ii), IL->hi(ii) - IL->lo(ii)); targets.push_back(new candidatePop(bubble, unitigs[targetID], IL->lo(ii), IL->hi(ii))); } delete IL; } targetIntervals.clear(); // If no targets, nothing to do. if (targets.size() == 0) continue; // Run through the placements again, and assign them to the correct target. // // For each read: // For each acceptable placement: // For each target location: // If the placement is for this target, save it. for (uint32 fi=0; fiufpath.size(); fi++) { uint32 readID = bubble->ufpath[fi].ident; for (uint32 pp=0; pptarget->id() == tid) && (targets[tt]->bgn < end) && (bgn < targets[tt]->end)) targets[tt]->placed.push_back(placed[readID][pp]); } } // Count the number of targets that have all the reads (later: in the correct order, etc, etc). Remove those // that don't. uint32 nTargets = 0; set tigReads; // Reads in the bubble tig. set tgtReads; // Reads in the bubble that have a placement in the target. // Remove duplicate placements from each target. for (uint32 tt=0; ttplaced.size(); aa++) { for (uint32 bb=0; bbplaced.size(); bb++) { if ((aa == bb) || (t->placed[aa].frgID != t->placed[bb].frgID) || (t->placed[aa].frgID == 0) || (t->placed[bb].frgID == 0)) continue; if (t->placed[aa].errors / t->placed[aa].aligned < t->placed[bb].errors / t->placed[bb].aligned) { #ifdef SHOW_MULTIPLE_PLACEMENTS writeLog("duplicate read alignment for tig %u read %u - better %u-%u %.4f - worse %u-%u %.4f\n", t->placed[aa].tigID, t->placed[aa].frgID, t->placed[aa].position.bgn, t->placed[aa].position.end, t->placed[aa].errors / t->placed[aa].aligned, t->placed[bb].position.bgn, t->placed[bb].position.end, t->placed[bb].errors / t->placed[bb].aligned); #endif t->placed[bb] = overlapPlacement(); } else { #ifdef SHOW_MULTIPLE_PLACEMENTS writeLog("duplicate read alignment for tig %u read %u - better %u-%u %.4f - worse %u-%u %.4f\n", t->placed[aa].tigID, t->placed[aa].frgID, t->placed[bb].position.bgn, t->placed[bb].position.end, t->placed[bb].errors / t->placed[bb].aligned, t->placed[aa].position.bgn, t->placed[aa].position.end, t->placed[aa].errors / t->placed[aa].aligned); #endif t->placed[aa] = overlapPlacement(); } } } // Get rid of any now-empty entries. for (uint32 aa=t->placed.size(); aa--; ) { if (t->placed[aa].frgID == 0) { t->placed[aa] = t->placed.back(); t->placed.pop_back(); } } } // Make a set of the reads in the bubble. We'll compare each target against this to decide if all reads are placed. for (uint32 fi=0; fiufpath.size(); fi++) tigReads.insert(bubble->ufpath[fi].ident); uint32 nOrphan = 0; // Full coverage; bubble can be popped. uint32 orphanTarget = 0; uint32 nBubble = 0; // Partial coverage, bubble cannot be popped. uint32 bubbleTarget = 0; for (uint32 tt=0; ttplaced.size(); op++) { if (logFileFlagSet(LOG_BUBBLE_DETAIL)) writeLog("tig %8u length %9u -> target %8u piece %2u position %9u-%9u length %8u - read %7u at %9u-%9u\n", bubble->id(), bubble->getLength(), targets[tt]->target->id(), tt, targets[tt]->bgn, targets[tt]->end, targets[tt]->end - targets[tt]->bgn, targets[tt]->placed[op].frgID, targets[tt]->placed[op].position.bgn, targets[tt]->placed[op].position.end); assert(targets[tt]->placed[op].frgID > 0); tgtReads.insert(targets[tt]->placed[op].frgID); } // Count the number of consecutive reads from the 5' or 3' end of the bubble that are placed // in the target. // // Also, count the number of reads in the bubble that are placed in the target. Likely the // same as n5 + n3. uint32 n5 = 0; uint32 n3 = 0; uint32 nt = 0; for (uint32 fi=0; fiufpath.size(); fi++) if (tgtReads.count(bubble->ufpath[fi].ident) > 0) n5++; else break; for (uint32 fi=bubble->ufpath.size(); fi-->0; ) if (tgtReads.count(bubble->ufpath[fi].ident) > 0) n3++; else break; for (uint32 fi=0; fiufpath.size(); fi++) if (tgtReads.count(bubble->ufpath[fi].ident) > 0) nt++; // Report now, before we nuke targets[tt] for being not a bubble! if ((nt == bubble->ufpath.size()) || ((n5 > 0) && (n3 > 0))) writeLog("tig %8u length %9u -> target %8u piece %2u position %9u-%9u length %8u - expected %3"F_SIZE_TP" reads, had %3"F_SIZE_TP" reads. n5=%3u n3=%3u nt=%3u\n", bubble->id(), bubble->getLength(), targets[tt]->target->id(), tt, targets[tt]->bgn, targets[tt]->end, targets[tt]->end - targets[tt]->bgn, tigReads.size(), tgtReads.size(), n5, n3, nt); // Decide if this is a bubble, orphan from construction, or repeat. if (nt == bubble->ufpath.size()) { nOrphan++; orphanTarget = tt; } else if ((n5 > 0) && (n3 > 0)) { nBubble++; bubbleTarget = tt; } } // If no placements, pbbbt. if (nOrphan + nBubble == 0) { //writeLog("tig %8u length %8u reads %6u had no bubble or orphan placements.\n", bubble->id(), bubble->getLength(), bubble->ufpath.size()); continue; } // If multiple orphan and/or bubble placements, it's a repeat. if (nOrphan + nBubble > 1) { writeLog("tig %8u length %8u reads %6u - repeat - %u orphan %u bubble placements.\n", bubble->id(), bubble->getLength(), bubble->ufpath.size(), nOrphan, nBubble); writeLog("\n"); bubble->_isRepeat = true; continue; } // If a bubble placement, mark it as a bubble so it can be skipped during repeat detection. if (nBubble > 0) { writeLog("tig %8u length %8u reads %6u - bubble\n", bubble->id(), bubble->getLength(), bubble->ufpath.size()); writeLog("\n"); bubble->_isBubble = true; continue; } // Otherwise, it's an orphan, move the reads to the proper place. writeLog("tig %8u length %8u reads %6u - orphan\n", bubble->id(), bubble->getLength(), bubble->ufpath.size()); for (uint32 op=0, tt=orphanTarget; opplaced.size(); op++) { ufNode frg; frg.ident = targets[tt]->placed[op].frgID; frg.contained = 0; frg.parent = 0; frg.ahang = 0; frg.bhang = 0; frg.position.bgn = targets[tt]->placed[op].position.bgn; frg.position.end = targets[tt]->placed[op].position.end; writeLog("move read %u from tig %u to tig %u %u-%u\n", frg.ident, bubble->id(), targets[tt]->target->id(), frg.position.bgn, frg.position.end); targets[tt]->target->addFrag(frg, 0, false); } writeLog("\n"); unitigs[bubble->id()] = NULL; delete bubble; } // Over all bubbles writeLog("\n"); // Needed if no bubbles are popped. delete [] placed; // Sort reads in all the tigs. Overkill, but correct. for (uint32 ti=0; tiufpath.size() == 1)) // Singleton, already sorted. continue; tig->sort(); } } canu-1.3/src/bogart/AS_BAT_PopBubbles.H000066400000000000000000000022271272335422300175750ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAR-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_BUBBLEPOPPING #define INCLUDE_AS_BAT_BUBBLEPOPPING #include "AS_global.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Unitig.H" void popBubbles(UnitigVector &unitigs, double deviationBubble); #endif // INCLUDE_AS_BAT_BUBBLEPOPPING canu-1.3/src/bogart/AS_BAT_PopBubbles.txt000066400000000000000000000067741272335422300202400ustar00rootroot00000000000000 findPotentialBubbles() - any unitig where at least half the reads an overlap to some other unitig is a candidate. - returns a map of unitig id (the bubble) to to a vector of unitig ids (the potential poppers). findBubbleReadPlacements() - threaded on the reads - for reads in potential bubbles, uses placeFragUsingOverlaps() to find high-quality alignments to unitigs that can pop the bubble. - returns an array of vector - one vector per read - of the placements for this read. Placements are high quality and to popper tigs only. popBubbles() - findPotentialBubbles() - findBubbleReadPlacements() - for each candidate tig: - build a map of unitig id (target) to an intervalList (targetIntervals) - add to the corresponding intervalList each bubble read placement, squish to intervals when done - filter out intervals that are too short (0.75x) or too long (1.25x) the bubble tig size - save size-consistent interavs to vector of candidatePop (targets) - clear targetIntervals list (its no longer needed) - for each read in the candidate tig - assign placements (from findBubbleReadPlacements()) to targets. some placements have no target - we now have a list of targets[] with: bubble*, target*, target bgn/end, vector of placed bubble reads in this region - decide if the candidate is a bubble, a repeat or an orphan - a bubble has the 5' and 3' most reads aligned, and only one target - a repeat has all reads aligned, and multiple targets - an orphan has all reads aligned, and one target ---------------------------------------- OLD-STYLE BUBBLE POPPING mergeSplitJoin() new intersectionList(unitigs) foreach unitig (NOT parallel) skip if fewer than 15 reads or 300 bases mergeBubbles() - based on previously discovered intersections stealBubbles() - nothing here, not implemented for each unitig (parallel) - unitigs created here are not reprocessed skip if fewer than 15 reads or 300 bases markRepeats() markChimera() ---------------------------------------- mergeBubbles(unitigs, erateBubble, targetUnitig, intersectionList) The intersection list is a 'reverse mapping of all BestEdges between unitigs'. For each read, a list of the incoming edges from other unitigs. foreach intersection point get potential bubble unitig if bubble unitig doesn't exist, it was popped already if bubble unitig is more than 500k, it is skipped if bubble unitig is the current unitig, it is skipped findEnds(), skip if none found checkEnds(), skip if bad checkFrags(), skip if fails bubble is merged, remove it findEnds() - return value is first/last reads find the first/last non-contained read get the correct edge get the unitig that edge points to discard the edge if the unitig it points to is the bubble (??) if both unitigs are null, return false checkEnds() - computes placement of first/last reads, false if inconsistent place both reads using overlaps find min/max coords of suspected correct placement if placedLength < bubbleLength / 2 -> return false, bubble shrank too much if placedLength > bubbleLength * 2 -> return false, bubble grew too much if first/last reads are the same, return true check order and orientation between bubble placement and popped placement bubble placed forward - reads have same orient and same order bubble placed reverse - reads have diff orient and diff order if so, return true return false checkFrags() - based on edges, we think the bubble goes here, try to place all the reads canu-1.3/src/bogart/AS_BAT_PopulateUnitig.C000066400000000000000000000156271272335422300205140ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_PopulateUnitig.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-19 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_PopulateUnitig.H" void populateUnitig(Unitig *unitig, BestEdgeOverlap *bestnext) { assert(unitig->getLength() > 0); if ((bestnext == NULL) || (bestnext->fragId() == 0)) // Nothing to add! return; ufNode frag = unitig->ufpath.back(); // The ID of the last fragment in the unitig, and the end we should walk off of it. int32 lastID = frag.ident; bool last3p = (frag.position.bgn < frag.position.end); uint32 nAdded = 0; // While there are fragments to add AND those fragments to add are not already in a unitig, // construct a reverse-edge, and add the fragment. while ((bestnext->fragId() != 0) && (Unitig::fragIn(bestnext->fragId()) == 0)) { BestEdgeOverlap bestprev; // Reverse nextedge (points from the unitig to the next fragment to add) so that it points from // the next fragment to add back to something in the unitig. If the fragments are // innie/outtie, we need to reverse the overlap to maintain that the A fragment is forward. if (last3p == bestnext->frag3p()) bestprev.set(lastID, last3p, bestnext->bhang(), bestnext->ahang(), bestnext->evalue()); else bestprev.set(lastID, last3p, -bestnext->ahang(), -bestnext->bhang(), bestnext->evalue()); // We just made 'bestprev' pointing from read 'bestnext->fragId()' end 'bestnext->frag3p()' // back to read 'lastID' end 'last3p'. Compute the placement. if (unitig->placeFrag(frag, bestnext->fragId(), bestnext->frag3p(), &bestprev)) { unitig->addFrag(frag, 0, false); nAdded++; } else { writeLog("ERROR: Failed to place frag %d into BOG path.\n", frag.ident); assert(0); } // Set up for the next fragmnet lastID = frag.ident; last3p = (frag.position.bgn < frag.position.end); bestnext = OG->getBestEdgeOverlap(lastID, last3p); } if (logFileFlagSet(LOG_BUILD_UNITIG)) if (bestnext->fragId() == 0) writeLog("Stopped adding at frag %u/%c' because no next best edge. Added %u reads.\n", lastID, (last3p) ? '3' : '5', nAdded); else writeLog("Stopped adding at frag %u/%c' beacuse next best frag %u/%c' is in unitig %u. Added %u reads.\n", lastID, (last3p) ? '3' : '5', bestnext->fragId(), bestnext->frag3p() ? '3' : '5', Unitig::fragIn(bestnext->fragId()), nAdded); } void populateUnitig(UnitigVector &unitigs, int32 fi) { if ((FI->fragmentLength(fi) == 0) || // Skip deleted (Unitig::fragIn(fi) != 0) || // Skip placed (OG->isContained(fi) == true)) // Skip contained return; Unitig *utg = unitigs.newUnitig(logFileFlagSet(LOG_BUILD_UNITIG)); // Add a first fragment -- to be 'compatable' with the old code, the first fragment is added // reversed, we walk off of its 5' end, flip it, and add the 3' walk. ufNode frag; frag.ident = fi; frag.contained = 0; frag.parent = 0; frag.ahang = 0; frag.bhang = 0; frag.position.bgn = FI->fragmentLength(fi); frag.position.end = 0; utg->addFrag(frag, 0, logFileFlagSet(LOG_BUILD_UNITIG)); // Add fragments as long as there is a path to follow...from the 3' end of the first fragment. BestEdgeOverlap *bestedge5 = OG->getBestEdgeOverlap(fi, false); BestEdgeOverlap *bestedge3 = OG->getBestEdgeOverlap(fi, true); assert(bestedge5->ahang() <= 0); // Best Edges must be dovetail, which makes this test assert(bestedge5->bhang() <= 0); // much simpler. assert(bestedge3->ahang() >= 0); assert(bestedge3->bhang() >= 0); // If this fragment is not covered by the two best overlaps we are finished. We will not follow // the paths out. This indicates either low coverage, or a chimeric fragment. If it is low // coverage, then the best overlaps will be mutual and we'll recover the same path. If it is a // chimeric fragment the overlaps will not be mutual and we will skip this fragment. // // The amount of our fragment that is covered by the two best overlaps is // // (fragLen + bestedge5->bhang()) + (fragLen - bestedge3->ahang()) // // If that is not significantly longer than the fragment length, then we will not use this // fragment as a seed for unitig construction. // if (OG->isSuspicious(fi)) return; #if 0 uint32 covered = FI->fragmentLength(fi) + bestedge5->bhang() + FI->fragmentLength(fi) - bestedge3->ahang(); // This breaks unitigs at 0x best-coverage regions. There might be a contain that spans (joins) // the two best overlaps to verify the fragment, but we can't easily tell right now. if (covered < FI->fragmentLength(fi) + AS_OVERLAP_MIN_LEN / 2) { writeLog("Stopping unitig construction of suspicious frag %d in unitig %d\n", utg->ufpath.back().ident, utg->id()); return; } #endif if (logFileFlagSet(LOG_BUILD_UNITIG)) writeLog("Adding 5' edges off of frag %d in unitig %d\n", utg->ufpath.back().ident, utg->id()); if (bestedge5->fragId()) populateUnitig(utg, bestedge5); utg->reverseComplement(false); if (logFileFlagSet(LOG_BUILD_UNITIG)) writeLog("Adding 3' edges off of frag %d in unitig %d\n", utg->ufpath.back().ident, utg->id()); if (bestedge3->fragId()) populateUnitig(utg, bestedge3); // Enabling this reverse complement is known to degrade the assembly. It is not known WHY it // degrades the assembly. // //utg->reverseComplement(false); } canu-1.3/src/bogart/AS_BAT_PopulateUnitig.H000066400000000000000000000031421272335422300205060ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_PopulateUnitig.H * * Modifications by: * * Brian P. Walenz from 2010-DEC-06 to 2013-AUG-01 * are Copyright 2010,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-19 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_POPULATEUNITIG #define INCLUDE_AS_BAT_POPULATEUNITIG void populateUnitig(Unitig *unitig, BestEdgeOverlap *nextedge); void populateUnitig(UnitigVector &unitigs, int32 fragID); #endif // INCLUDE_AS_BAT_POPULATUNITIG canu-1.3/src/bogart/AS_BAT_PromoteToSingleton.C000066400000000000000000000043331272335422300213460ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_PromoteToSingleton.C * * Modifications by: * * Brian P. Walenz from 2012-JAN-05 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-19 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_Unitig.H" // If we are not reconstructing repeats, promote all the unplaced fragments to new unitigs. // Oodles of possibilities here; promote everything to a singleton unitig, promote only // the non-contained, then place contains, then promote what is left over, etc. void promoteToSingleton(UnitigVector &unitigs) { for (uint32 fi=1; fi<=FI->numFragments(); fi++) { if (Unitig::fragIn(fi) != 0) // Placed already continue; if (FI->fragmentLength(fi) == 0) // Deleted. continue; Unitig *utg = unitigs.newUnitig(false); ufNode frag; frag.ident = fi; frag.contained = 0; frag.parent = 0; frag.ahang = 0; frag.bhang = 0; frag.position.bgn = 0; frag.position.end = FI->fragmentLength(fi); utg->addFrag(frag, 0, false); } } canu-1.3/src/bogart/AS_BAT_PromoteToSingleton.H000066400000000000000000000021241272335422300213470ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * bogart/AS_BAT_BreakRepeats.H * * Modifications by: * * Brian P. Walenz beginning on 2016-APR-13 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_PROMOTE_TO_SINGLETON void promoteToSingleton(UnitigVector &unitigs); #endif // INCLUDE_AS_BAT_PROMOTE_TO_SINGLETON canu-1.3/src/bogart/AS_BAT_ReconstructRepeats.C000066400000000000000000000054631272335422300213770ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_ReconstructRepeats.C * * Modifications by: * * Brian P. Walenz from 2012-JAN-05 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2015-APR-24 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_ChunkGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_PlaceFragUsingOverlaps.H" #include "AS_BAT_PopulateUnitig.H" #include "AS_BAT_PlaceContains.H" // estimate read error rate from best overlaps (per library?) // use that error rate below when rebuilding repeats void reconstructRepeats(UnitigVector &unitigs, double erateGraph, double deviationGraph) { // Build a set<> of all the unplaced fragments, then construct a new BOG and CG from which we // construct unitigs. BestOverlapGraph *OGsave = OG; ChunkGraph *CGsave = CG; set unplaced; for (uint32 fi=1; fi<=FI->numFragments(); fi++) if (Unitig::fragIn(fi) == 0) unplaced.insert(fi); OG = new BestOverlapGraph(erateGraph / 2.0, deviationGraph, &unplaced); CG = new ChunkGraph(&unplaced); writeLog("==> BUILDING REPEAT UNITIGS from %d fragments.\n", unplaced.size()); for (uint32 fi=CG->nextFragByChunkLength(); fi>0; fi=CG->nextFragByChunkLength()) populateUnitig(unitigs, fi); writeLog("==> BUILDING REPEAT UNITIGS catching missed fragments.\n"); for (uint32 fi=1; fi <= FI->numFragments(); fi++) populateUnitig(unitigs, fi); writeLog("==> BUILDING REPEAT UNITIGS placing contained fragments.\n"); placeUnplacedUsingAllOverlaps(unitigs, "PREFIX"); delete OG; delete CG; OG = OGsave; CG = CGsave; } canu-1.3/src/bogart/AS_BAT_SetParentAndHang.C000066400000000000000000000131451272335422300206620ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_SetParentAndHang.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010,2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2015-AUG-05 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_OverlapCache.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_UnitigVector.H" #include "AS_BAT_SetParentAndHang.H" void setParentAndHang(UnitigVector &unitigs) { return; map forward; map allreads; // Just for stats, build a map fo the reads in the unitig. for (uint32 ti=0; tiufpath.size() == 0) continue; // Reset parent and hangs, build a map of the reads in the unitig. for (uint32 fi=1; fiufpath.size(); fi++) { ufNode *frg = &tig->ufpath[fi]; frg->parent = 0; frg->ahang = 0; frg->bhang = 0; allreads[frg->ident] = true; } // For each fragment, set parent/hangs using the edges. for (uint32 fi=0; fiufpath.size(); fi++) { ufNode *frg = &tig->ufpath[fi]; // Remember that we've placed this read, and if it was forward or reverse. forward[frg->ident] = (frg->position.bgn < frg->position.end); // If the first read, there is no parent possible. if (ti == 0) continue; // Otherwise, find the thickest overlap to any read already placed in the unitig. uint32 olapsLen = 0; BAToverlap *olaps = OC->getOverlaps(frg->ident, AS_MAX_EVALUE, olapsLen); uint32 tt = UINT32_MAX; uint32 ttLen = 0; double ttErr = DBL_MAX; int32 ah = 0; int32 bh = 0; uint32 notPresent = 0; // Potential parent isn't in the unitig uint32 notPlaced = 0; // Potential parent isn't placed yet uint32 negHang = 0; // Potential parent has a negative hang to a placed read uint32 goodOlap = 0; for (uint32 oo=0; oooverlapLength(olaps[oo].a_iid, olaps[oo].b_iid, olaps[oo].a_hang, olaps[oo].b_hang); // Compute the hangs, so we can ignore those that would place this read before the parent. // This is a flaw somewhere in bogart, and should be caught and fixed earlier. // Consensus is expecting the have the hangs for the parent read, not this read, and some // fiddling is needed to flip the overlap for this: // First, swap the reads so it's b-vs-a. // Then, flip the overlap if the b read is in the unitig flipped. int32 ah = (olaps[oo].flipped == false) ? (-olaps[oo].a_hang) : (olaps[oo].b_hang); int32 bh = (olaps[oo].flipped == false) ? (-olaps[oo].b_hang) : (olaps[oo].a_hang); if (forward[olaps[oo].b_iid] == false) { swap(ah, bh); ah = -ah; bh = -bh; } // If the ahang is negative, we flubbed up somewhere, and want to place this read before // the parent (even though positions say to place it after, because we sorted by position). if (ah < 0) { //fprintf(stderr, "ERROR: read %u in tig %u has negative ahang from parent read %u, ejected.\n", // frg->ident, ti, olaps[oo].b_iid); negHang++; continue; } // The overlap is good. Count it as such. goodOlap++; // If the overlap is worse than the one we already have, we don't care. if ((l < ttLen) || // Too short (ttErr < olaps[oo].erate)) { // Too noisy continue; } tt = oo; ttLen = l; ttErr = olaps[oo].erate; } // If no thickest overlap, we screwed up somewhere. Complain and eject the read. if (tt == UINT32_MAX) { fprintf(stderr, "ERROR: read %u in tig %u has no overlap to any previous read, ejected. %u overlaps total. %u negative hang. %u to read not in tig. %u to read later in tig. %u good overlaps.\n", frg->ident, tig->tigID(), olapsLen, negHang, notPresent, notPlaced, goodOlap); continue; } frg->parent = olaps[tt].b_iid; frg->ahang = ah; frg->bhang = bh; } // Over all fragments } // Over all unitigs } canu-1.3/src/bogart/AS_BAT_SetParentAndHang.H000066400000000000000000000027631272335422300206730ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_SetParentAndHang.H * * Modifications by: * * Brian P. Walenz from 2010-DEC-06 to 2013-AUG-01 * are Copyright 2010,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-19 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_SETPARENTANDHANG #define INCLUDE_AS_BAT_SETPARENTANDHANG #include "AS_BAT_UnitigVector.H" void setParentAndHang(UnitigVector &unitigs); #endif // INCLUDE_AS_BAT_SETPARENTANDHANG canu-1.3/src/bogart/AS_BAT_SplitDiscontinuous.C000066400000000000000000000145221272335422300214160ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_SplitDiscontinuous.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2015-MAR-03 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" static void makeNewUnitig(UnitigVector &unitigs, uint32 splitFragsLen, ufNode *splitFrags) { Unitig *dangler = unitigs.newUnitig(false); if (logFileFlagSet(LOG_SPLIT_DISCONTINUOUS)) writeLog("splitDiscontinuous()-- new tig "F_U32" with "F_U32" fragments (starting at frag "F_U32").\n", dangler->id(), splitFragsLen, splitFrags[0].ident); int splitOffset = -MIN(splitFrags[0].position.bgn, splitFrags[0].position.end); // This should already be true, but we force it still splitFrags[0].contained = 0; for (uint32 i=0; iaddFrag(splitFrags[i], splitOffset, false); //logFileFlagSet(LOG_SPLIT_DISCONTINUOUS)); } // After splitting and ejecting some contains, check for discontinuous unitigs. // void splitDiscontinuousUnitigs(UnitigVector &unitigs, uint32 minOverlap) { writeLog("==> SPLIT DISCONTINUOUS\n"); uint32 numTested = 0; uint32 numSplit = 0; uint32 numCreated = 0; uint32 splitFragsLen = 0; uint32 splitFragsMax = 0; ufNode *splitFrags = NULL; for (uint32 ti=0; tiufpath.size() < 2)) continue; // Unitig must be sorted. Someone upstream os screwing this up. tig->sort(); // We'll want to build an array of new fragments to split out. This can be up // to the size of the largest unitig. splitFragsMax = MAX(splitFragsMax, tig->ufpath.size()); // Check that the unitig starts at position zero. Not critical for the next loop, but // needs to be dome sometime. int32 minPos = MIN(tig->ufpath[0].position.bgn, tig->ufpath[0].position.end); if (minPos == 0) continue; writeLog("splitDiscontinuous()-- tig "F_U32" offset messed up; reset by "F_S32".\n", tig->id(), minPos); for (uint32 fi=0; fiufpath.size(); fi++) { ufNode *frg = &tig->ufpath[fi]; frg->position.bgn -= minPos; frg->position.end -= minPos; } } splitFrags = new ufNode [splitFragsMax]; // Now, finally, we can check for gaps in unitigs. for (uint32 ti=0; tiufpath.size() < 2)) continue; // We don't expect many unitigs to be broken, so we'll do a first quick pass to just // test if it is. int32 maxEnd = MAX(tig->ufpath[0].position.bgn, tig->ufpath[0].position.end); bool isBroken = false; for (uint32 fi=0; fiufpath.size(); fi++) { ufNode *frg = &tig->ufpath[fi]; int32 bgn = MIN(frg->position.bgn, frg->position.end); int32 end = MAX(frg->position.bgn, frg->position.end); if (bgn > maxEnd - minOverlap) { isBroken = true; break; } maxEnd = MAX(maxEnd, end); } numTested++; if (isBroken == false) continue; numSplit++; // Dang, busted unitig. Fix it up. splitFragsLen = 0; maxEnd = 0; if (logFileFlagSet(LOG_SPLIT_DISCONTINUOUS)) writeLog("splitDiscontinuous()-- discontinuous tig "F_U32" with "F_SIZE_T" fragments broken into:\n", tig->id(), tig->ufpath.size()); for (uint32 fi=0; fiufpath.size(); fi++) { ufNode *frg = &tig->ufpath[fi]; int32 bgn = MIN(frg->position.bgn, frg->position.end); int32 end = MAX(frg->position.bgn, frg->position.end); // Good thick overlap exists to this fragment, save it. if (bgn <= maxEnd - minOverlap) { assert(splitFragsLen < splitFragsMax); splitFrags[splitFragsLen++] = *frg; maxEnd = MAX(maxEnd, end); continue; } // No thick overlap found. We need to break right here before the current fragment. We used // to try to place contained reads with their container. For simplicity, we instead just // make a new unitig, letting the main() decide what to do with them (e.g., bubble pop or try // to place all reads in singleton unitigs as contained reads again). numCreated++; makeNewUnitig(unitigs, splitFragsLen, splitFrags); tig = unitigs[ti]; // Done with the split, save the current fragment. This resets everything. splitFragsLen = 0; splitFrags[splitFragsLen++] = *frg; maxEnd = end; } // If we did any splitting, then the length of the frags in splitFrags will be less than the length // of the path in the current unitig. Make a final new unitig for the remaining fragments. // if (splitFragsLen != tig->ufpath.size()) { numCreated++; makeNewUnitig(unitigs, splitFragsLen, splitFrags); delete unitigs[ti]; unitigs[ti] = NULL; } } writeLog("splitDiscontinuous()-- Tested "F_U32" unitigs, split "F_U32" into "F_U32" new unitigs.\n", numTested, numSplit, numCreated); delete [] splitFrags; } canu-1.3/src/bogart/AS_BAT_SplitDiscontinuous.H000066400000000000000000000030131272335422300214140ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_SplitDiscontinuous.H * * Modifications by: * * Brian P. Walenz from 2010-DEC-06 to 2013-AUG-01 * are Copyright 2010,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2015-MAR-03 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_SPLITDISCONTINUOUS #define INCLUDE_AS_BAT_SPLITDISCONTINUOUS void splitDiscontinuousUnitigs(UnitigVector &unitigs, uint32 minOverlap); #endif // INCLUDE_AS_BAT_SPLITDISCONTINUOUS canu-1.3/src/bogart/AS_BAT_Unitig.C000066400000000000000000000352131272335422300167730ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_Unitig.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-19 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_BestOverlapGraph.H" static std::map* containPartialOrder; uint32* Unitig::_inUnitig = NULL; uint32* Unitig::_pathPosition = NULL; #undef SHOW_PROFILE_CONSTRUCTION #undef SHOW_PROFILE_CONSTRUCTION_DETAILS void Unitig::reverseComplement(bool doSort) { // If there are contained fragments, we need to sort by position to place them correctly after // their containers. If there are no contained fragments, sorting can break the initial unitig // building. When two frags start at position zero, we'll exchange the order. Initial unitig // building depends on having the first fragment added become the last fragment in the unitig // after reversing. for (uint32 fi=0; fiposition.bgn = getLength() - frg->position.bgn; frg->position.end = getLength() - frg->position.end; //if (frg->contained != 0) // doSort = true; assert(frg->position.bgn >= 0); assert(frg->position.end >= 0); } // We've updated the positions of everything. Now, sort or reverse the list, and rebuild the // pathPosition map. if (doSort) { sort(); } else { std::reverse(ufpath.begin(), ufpath.end()); for (uint32 fi=0; fi *hist) { sort(); for (uint32 fi=0; fiposition.bgn < rdA->position.end); int32 rdAlo = (rdAfwd) ? rdA->position.bgn : rdA->position.end; int32 rdAhi = (rdAfwd) ? rdA->position.end : rdA->position.bgn; for (uint32 fj=1; fj<6; fj++) { if (fi + fj < ufpath.size()) { ufNode *rdB = &ufpath[fi+fj]; bool rdBfwd = (rdB->position.bgn < rdB->position.end); int32 rdBlo = (rdBfwd) ? rdB->position.bgn : rdB->position.end; int32 rdBhi = (rdBfwd) ? rdB->position.end : rdB->position.bgn; uint32 dist = rdBlo - rdAlo; hist[fj].push_back(dist); } } } } #if 1 void Unitig::computeErrorProfileApproximate(const char *UNUSED(prefix), const char *UNUSED(label)) { } #endif void Unitig::computeErrorProfile(const char *UNUSED(prefix), const char *UNUSED(label)) { #ifdef SHOW_PROFILE_CONSTRUCTION writeLog("Find error profile for tig "F_U32" of length "F_U32" with "F_SIZE_T" reads.\n", id(), getLength(), ufpath.size()); #endif errorProfile.clear(); errorProfileIndex.clear(); vector olaps; // Pick a set of reads to use. We need full coverage in overlaps. // Scan overlaps to find those that we care about, and save their endpoints. for (uint32 fi=0; fiposition.bgn < rdA->position.end); int32 rdAlo = (rdAfwd) ? rdA->position.bgn : rdA->position.end; int32 rdAhi = (rdAfwd) ? rdA->position.end : rdA->position.bgn; uint32 ovlLen = 0; BAToverlap *ovl = OC->getOverlaps(rdA->ident, AS_MAX_ERATE, ovlLen); uint32 nDiffTig = 0; uint32 nDiffPos = 0; uint32 nIsect = 0; for (uint32 oi=0; oiposition.bgn < rdB->position.end); int32 rdBlo = (rdBfwd) ? rdB->position.bgn : rdB->position.end; int32 rdBhi = (rdBfwd) ? rdB->position.end : rdB->position.bgn; if ((rdAhi < rdBlo) || (rdBhi < rdAlo)) { nDiffPos++; #ifdef SHOW_PROFILE_CONSTRUCTION_DETAILS writeLog("diffPos rdA %u=%u %u-%u rdB %u=%u %u-%u\n", ovl[oi].a_iid, rdA->ident, rdAlo, rdAhi, ovl[oi].b_iid, rdB->ident, rdBlo, rdBhi); #endif continue; } // Now figure out what region is covered by the overlap. int32 tiglo = 0; int32 tighi = FI->fragmentLength(rdA->ident); if (ovl[oi].a_hang > 0) tiglo += ovl[oi].a_hang; // Postiive hang! if (ovl[oi].b_hang < 0) tighi += ovl[oi].b_hang; // Negative hang! assert(0 <= tiglo); assert(0 <= tighi); assert(tiglo <= tighi); assert(tiglo <= FI->fragmentLength(rdA->ident)); assert(tighi <= FI->fragmentLength(rdA->ident)); // Offset and adjust to tig coordinates // Beacuse the read is placed with a lot of fudging in the positions, we need // to scale the coordinates we compute here. double sc = (rdAhi - rdAlo) / (double)FI->fragmentLength(rdA->ident); uint32 bgn = (uint32)floor(rdAlo + sc * tiglo); uint32 end = (uint32)floor(rdAlo + sc * tighi); nIsect++; olaps.push_back(epOlapDat(bgn, true, ovl[oi].erate)); olaps.push_back(epOlapDat(end, false, ovl[oi].erate)); } #ifdef SHOW_PROFILE_CONSTRUCTION_DETAILS writeLog("tig %u read %u with %u overlaps - diffTig %u diffPos %u intersect %u\n", id(), rdA->ident, ovlLen, nDiffTig, nDiffPos, nIsect); #endif } #ifdef SHOW_PROFILE_CONSTRUCTION writeLog("tig %u generated "F_SIZE_T" olaps.\n", id(), olaps.size()); #endif // Sort. std::sort(olaps.begin(), olaps.end()); // Convert coordinates into intervals. Conceptually, squish out the duplicate numbers, then // create an interval for every adjacent pair. We need to add intervals for the first and last // region. And one more, for convenience, to hold the final 'close' values on intervals that // extend to the end of the unitig. if (olaps[0].pos != 0) errorProfile.push_back(epValue(0, olaps[0].pos)); for (uint32 bb=0, ii=1; ii curDev; for (uint32 oo=0, ee=0; oo bgn)) { fprintf(stderr, "BAD ESTIMATE for bgn=%u end=%u\n", bgn, end); pbi--; } while ((pbi < errorProfileIndex.size()) && (errorProfile[errorProfileIndex[pbi]].end <= bgn)) pbi++; if (pbi == errorProfileIndex.size()) { //fprintf(stderr, "Fell off loop for bgn=%u end=%u last ep bgn=%u end=%u\n", // bgn, end, errorProfile.back().bgn, errorProfile.back().end); pbi--; } // The region pb points to will contain bgn. uint32 pb = errorProfileIndex[pbi]; //fprintf(stderr, "For bgn=%u end=%u - stopped at pbi=%u errorProfile[%u] = %u-%u (1)\n", // bgn, end, pbi, pb, errorProfile[pb].bgn, errorProfile[pb].end); // Fine tune search to find the exact first region. while ((0 < pb) && (bgn < errorProfile[pb].bgn)) pb--; while ((pb < errorProfile.size()) && (errorProfile[pb].end <= bgn)) pb++; #endif if ((errorProfile[pb].bgn > bgn) || (bgn >= errorProfile[pb].end)) fprintf(stderr, "For bgn=%u end=%u - stopped at errorProfile[%u] = %u-%u BOOM\n", bgn, end, pb, errorProfile[pb].bgn, errorProfile[pb].end); assert(errorProfile[pb].bgn <= bgn); assert(bgn < errorProfile[pb].end); // Sum the number of bases above the supplied erate. uint32 pe = pb; while ((pe < errorProfile.size()) && (errorProfile[pe].bgn < end)) { if (erate <= errorProfile[pe].max(deviations)) nAbove += errorProfile[pe].end - errorProfile[pe].bgn; else nBelow += errorProfile[pe].end - errorProfile[pe].bgn; pe++; } // Adjust for the bits we overcounted in the first and last regions. if (pe > 0) // Argh. If this read is fully in the first region (where there pe--; // is only 1x coverage) then pe==0. uint32 bb = bgn - errorProfile[pb].bgn; uint32 be = errorProfile[pe].end - end; assert(bgn >= errorProfile[pb].bgn); assert(errorProfile[pe].end >= end); if (erate <= errorProfile[pb].max(deviations)) nAbove -= bb; else nBelow -= bb; if (erate <= errorProfile[pe].max(deviations)) nAbove -= be; else nBelow -= be; assert(nAbove >= 0); assert(nBelow >= 0); return((double)nAbove / (nBelow + nAbove)); } void Unitig::reportErrorProfile(const char *prefix, const char *label) { char N[FILENAME_MAX]; FILE *F; sprintf(N, "%s.%s.%08u.profile", prefix, label, id()); F = fopen(N, "w"); if (F) { for (uint32 ii=0; ii #include class BestEdgeOverlap; class SeqInterval { public: SeqInterval() { bgn = 0; end = 0; }; ~SeqInterval() { }; int32 min(void) const { return(::min(bgn, end)); }; int32 max(void) const { return(::max(bgn, end)); }; bool isForward(void) const { return(bgn < end); }; bool isReverse(void) const { return(bgn > end); }; bool operator==(SeqInterval const that) const { return(((bgn == that.bgn) && (end == that.end)) || ((bgn == that.end) && (end == that.bgn))); }; bool operator!=(SeqInterval const that) const { return(((bgn != that.bgn) || (end != that.end)) && ((bgn != that.end) || (end != that.bgn))); }; bool operator<(SeqInterval const that) const { return(min() < that.min()); #if 0 if (isReverse()) { if (b.isReverse()) return end < that.end; else return end < that.bgn; } else { if (b.isReverse()) return bgn < that.end; else return bgn < that.bgn; } #endif }; public: int32 bgn; // MUST be signed! Read placement needs to set coordinates to negative int32 end; // coordinates to indicate the read extends off the start of the tig. }; // Derived from IntMultiPos, but removes some of the data (48b in IntMultiPos, 32b in struct // ufNode). The minimum size (bit fields, assuming maximum limits, not using the contained // field) seems to be 24b, and is more effort than it is worth (just removing 'contained' would be // a chore). // // ufNode is, of course, 'unitig fragment node'. // class ufNode { public: uint32 ident; uint32 contained; uint32 parent; // IID of the fragment we align to int32 ahang; // If parent defined, these are relative int32 bhang; // that fragment SeqInterval position; bool isForward(void) const { return(position.isForward()); }; bool isReverse(void) const { return(position.isReverse()); }; bool operator<(ufNode const &that) const { int32 abgn = (position.bgn < position.end) ? position.bgn : position.end; int32 aend = (position.bgn < position.end) ? position.end : position.bgn; int32 bbgn = (that.position.bgn < that.position.end) ? that.position.bgn : that.position.end; int32 bend = (that.position.bgn < that.position.end) ? that.position.end : that.position.bgn; if (abgn < bbgn) return(true); // A starts before B! if (abgn > bbgn) return(false); // B starts before A! if (aend < bend) return(false); // A contained in B, not less than. if (aend > bend) return(true); // B contained in A, is less than. return(false); // Equality, not less than. }; }; class Unitig { private: Unitig() { _length = 0; _id = 0; _tigID = 0; _isUnassembled = false; _isBubble = false; _isRepeat = false; _isCircular = false; }; public: ~Unitig(void) { }; friend class UnitigVector; void sort(void) { std::sort(ufpath.begin(), ufpath.end()); for (uint32 fi=0; fi dev; }; static size_t epValueSize(void) { return(sizeof(epValue)); }; void computeArrivalRate(const char *prefix, const char *label, vector *hist); void computeErrorProfileApproximate(const char *prefix, const char *label); void computeErrorProfile(const char *prefix, const char *label); void reportErrorProfile(const char *prefix, const char *label); void clearErrorProfile(void) { errorProfile.clear(); }; double overlapConsistentWithTig(double deviations, uint32 bgn, uint32 end, double erate); // Returns the read that is touching the start of the tig. ufNode *firstRead(void) { ufNode *rd5 = &ufpath.front(); for (uint32 fi=1; (fi < ufpath.size()) && (rd5->position.min() != 0); fi++) rd5 = &ufpath[fi]; assert(rd5->position.min() == 0); return(rd5); }; // Returns the read that is touching the end of the tig. ufNode *lastRead(void) { ufNode *rd3 = &ufpath.back(); for (uint32 fi=ufpath.size()-1; (fi-- > 0) && (rd3->position.max() != getLength()); ) rd3 = &ufpath[fi]; assert(rd3->position.max() == getLength()); return(rd3); }; static void removeFrag(int32 fid) { _inUnitig[fid] = 0; _pathPosition[fid] = ~0; }; static uint32 fragIn(uint32 fragId) { if ((_inUnitig == NULL) || (fragId == 0)) return 0; return _inUnitig[fragId]; }; static uint32 pathPosition(uint32 fragId) { if ((_pathPosition == NULL) || (fragId == 0)) return ~0; return _pathPosition[fragId]; }; static void resetFragUnitigMap(uint32 numFrags) { if (_inUnitig == NULL) _inUnitig = new uint32[numFrags+1]; memset(_inUnitig, 0, (numFrags+1) * sizeof(uint32)); if (_pathPosition == NULL) _pathPosition = new uint32[numFrags+1]; memset(_pathPosition, 0, (numFrags+1) * sizeof(uint32)); }; // Public Member Variables vector ufpath; vector errorProfile; vector errorProfileIndex; private: int32 _length; uint32 _id; public: uint32 _tigID; private: static uint32 *_inUnitig; // Maps a fragment iid to a unitig id. static uint32 *_pathPosition; // Maps a fragment iid to an index in the dovetail path public: // Classification. The output is in three files: 'unassembled', 'bubbles', 'contigs' (defined as // not unassembled and not bubble). uint32 _isUnassembled; // Is a single read or a pseudo singleton uint32 _isBubble; // Annotation: from a failed bubble pop uint32 _isRepeat; // Annotation: from an identified repeat region uint32 _isCircular; // Annotation: has overlap to self char type(void) { if (_isUnassembled) return('U'); if (_isBubble) return('B'); if (_isRepeat) return('R'); if (_isCircular) return('C'); return('N'); } }; #endif // INCLUDE_AS_BAT_UNITIG canu-1.3/src/bogart/AS_BAT_UnitigVector.C000066400000000000000000000123341272335422300201550ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-APR-06 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_UnitigVector.H" UnitigVector::UnitigVector() { _blockSize = 1048576; _numBlocks = 1; _maxBlocks = 1024; _blocks = new Unitig ** [_maxBlocks]; _blocks[0] = new Unitig * [_blockSize]; _blocks[0][0] = NULL; // No first unitig. _blockNext = 1; _totalUnitigs = 1; }; UnitigVector::~UnitigVector() { // Delete the unitigs. for (uint32 ii=0; ii<_numBlocks; ii++) for (uint32 jj=0; jj<_blockSize; jj++) delete _blocks[ii][jj]; // Delete the blocks. for (uint32 ii=0; ii<_numBlocks; ii++) delete [] _blocks[ii]; // And the block pointers. delete [] _blocks; }; Unitig * UnitigVector::newUnitig(bool verbose) { Unitig *u = new Unitig(); #pragma omp critical { u->_id = _totalUnitigs++; if (verbose) writeLog("Creating Unitig %d\n", u->_id); if (_blockNext >= _blockSize) { assert(_numBlocks < _maxBlocks); _blocks[_numBlocks] = new Unitig * [_blockSize]; memset(_blocks[_numBlocks], 0, sizeof(Unitig **) * _blockSize); _numBlocks++; _blockNext = 0; } _blocks[_numBlocks-1][_blockNext++] = u; // The rest are just sanity checks. assert((u->id() / _blockSize) == (_numBlocks - 1)); assert((u->id() % _blockSize) == (_blockNext - 1)); assert(operator[](u->id()) == u); } return(u); }; void UnitigVector::deleteUnitig(uint32 i) { delete _blocks[i / _blockSize][i % _blockSize]; _blocks[i / _blockSize][i % _blockSize] = NULL; } #ifdef CHECK_UNITIG_ARRAY_INDEXING Unitig *&operator[](uint32 i) { uint32 idx = i / _blockSize; uint32 pos = i % _blockSize; if (((i >= _totalUnitigs)) || ((idx >= _numBlocks)) || (((pos >= _blockNext) && (idx >= _numBlocks - 1)))) { fprintf(stderr, "UnitigVector::operator[]()-- i="F_U32" with totalUnitigs="F_U64"\n", i, _totalUnitigs); fprintf(stderr, "UnitigVector::operator[]()-- blockSize="F_U64"\n", _blockSize); fprintf(stderr, "UnitigVector::operator[]()-- idx="F_U32" numBlocks="F_U64"\n", idx, _numBlocks); fprintf(stderr, "UnitigVector::operator[]()-- pos="F_U32" blockNext="F_U64"\n", pos, _blockNext); } assert(i < _totalUnitigs); assert((idx < _numBlocks)); assert((pos < _blockNext) || (idx < _numBlocks - 1)); return(_blocks[idx][pos]); }; #endif void UnitigVector::computeArrivalRate(const char *prefix, const char *label) { uint32 tiLimit = size(); uint32 numThreads = omp_get_max_threads(); uint32 blockSize = (tiLimit < 100000 * numThreads) ? numThreads : tiLimit / 99999; fprintf(stderr, "Computing arrival rates for %u unitigs using %u threads.\n", tiLimit, numThreads); vector hist[6]; //#pragma omp parallel for schedule(dynamic, blockSize) for (uint32 ti=0; tiufpath.size() == 1) continue; tig->computeArrivalRate(prefix, label, hist); } for (uint32 ii=1; ii<6; ii++) { char N[FILENAME_MAX]; sprintf(N, "%s.arrivalRate.%u.dat", prefix, ii); FILE *F = fopen(N, "w"); for (uint32 jj=0; jjufpath.size() == 1) continue; tig->computeErrorProfile(prefix, label); } fprintf(stderr, "Computing error profiles - FINISHED.\n"); } void UnitigVector::reportErrorProfiles(const char *prefix, const char *label) { uint32 tiLimit = size(); uint32 numThreads = omp_get_max_threads(); uint32 blockSize = (tiLimit < 100000 * numThreads) ? numThreads : tiLimit / 99999; for (uint32 ti=0; tiufpath.size() == 1) continue; tig->reportErrorProfile(prefix, label); } } canu-1.3/src/bogart/AS_BAT_UnitigVector.H000066400000000000000000000033671272335422300201700ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-APR-06 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_AS_BAT_UNITIG_VECTOR #define INCLUDE_AS_BAT_UNITIG_VECTOR #include "AS_global.H" class Unitig; class UnitigVector { public: UnitigVector(); ~UnitigVector(); Unitig *newUnitig(bool verbose); void deleteUnitig(uint32 i); size_t size(void) { return(_totalUnitigs); }; Unitig *&operator[](uint32 i) { return(_blocks[i / _blockSize][i % _blockSize]); }; void computeArrivalRate(const char *prefix, const char *label); void computeErrorProfiles(const char *prefix, const char *label); void reportErrorProfiles(const char *prefix, const char *label); private: uint64 _blockSize; uint64 _numBlocks; uint64 _maxBlocks; Unitig ***_blocks; uint64 _blockNext; uint64 _totalUnitigs; }; #endif // INCLUDE_AS_BAT_UNITIG_VECTOR canu-1.3/src/bogart/AS_BAT_Unitig_AddFrag.C000066400000000000000000000066461272335422300203530ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_Unitig_AddFrag.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010,2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-19 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" void Unitig::addFrag(ufNode node, int offset, bool report) { node.position.bgn += offset; node.position.end += offset; assert(node.ident > 0); // keep track of the unitig a frag is in _inUnitig[node.ident] = _id; _pathPosition[node.ident] = ufpath.size(); // keep track of max position in unitig int32 frgEnd = MAX(node.position.bgn, node.position.end); if (frgEnd > _length) _length = frgEnd; ufpath.push_back(node); if ((report) || (node.position.bgn < 0) || (node.position.end < 0)) { int32 trulen = FI->fragmentLength(node.ident); int32 poslen = (node.position.end > node.position.bgn) ? (node.position.end - node.position.bgn) : (node.position.bgn - node.position.end); if (node.contained) writeLog("Added frag %d (len %d) to unitig %d at %d,%d (idx %lu) (lendiff %d) (contained in %d)\n", node.ident, trulen, _id, node.position.bgn, node.position.end, ufpath.size() - 1, poslen - trulen, node.contained); else writeLog("Added frag %d (len %d) to unitig %d at %d,%d (idx %lu) (lendiff %d)\n", node.ident, trulen, _id, node.position.bgn, node.position.end, ufpath.size() - 1, poslen - trulen); assert(poslen / trulen < 10); assert(trulen / poslen < 10); } assert(node.position.bgn >= 0); assert(node.position.end >= 0); } // Percolate the last fragment to the correct spot in the list. #if 0 void Unitig::bubbleSortLastFrag(void) { uint32 previd = ufpath.size() - 2; uint32 lastid = ufpath.size() - 1; ufNode last = ufpath[lastid]; uint32 lastbgn = MIN(last.position.bgn, last.position.end); while ((lastid > 0) && (lastbgn < MIN(ufpath[previd].position.bgn, ufpath[previd].position.end))) { ufpath[lastid] = ufpath[previd]; _pathPosition[ufpath[lastid].ident] = lastid; lastid--; previd--; } _pathPosition[last.ident] = lastid; if (lastid < ufpath.size() - 1) ufpath[lastid] = last; } #endif canu-1.3/src/bogart/AS_BAT_Unitig_PlaceFragUsingEdges.C000066400000000000000000000240431272335422300226540ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_Unitig_PlaceFragUsingEdges.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2015-MAR-06 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #undef DEBUG_PLACE_FRAG ufNode placeFrag_contained(uint32 fragId, ufNode &parent, BestEdgeOverlap *edge) { bool pFwd = (parent.position.bgn < parent.position.end) ? true : false; int32 pMin = (parent.position.bgn < parent.position.end) ? parent.position.bgn : parent.position.end; int32 pMax = (parent.position.bgn < parent.position.end) ? parent.position.end : parent.position.bgn; assert(pMin < pMax); // Reverse the overlap. frag3p here means the overlap is flipped. int32 ahang = (edge->frag3p() == false) ? -edge->ahang() : edge->bhang(); int32 bhang = (edge->frag3p() == false) ? -edge->bhang() : edge->ahang(); // Depending on the parent orientation... // // pMin pMax pMin pMax // ----------------> <---------------- // ahang ----- bhang bhang ----- ahang // > 0 < 0 < 0 > 0 int32 fMin = (pFwd == true) ? pMin + ahang : pMin - bhang; int32 fMax = (pFwd == true) ? pMax + bhang : pMax - ahang; //int32 fMin = pMin + ((frag3p == false) ? -edge->ahang() : edge->bhang()); // * intraScale //int32 fMax = pMax + ((frag3p == false) ? -edge->bhang() : edge->ahang()); // * interScale assert(fMin < fMax); // We don't know the true length of the overlap, and our hang-based math tends to shrink reads. // Reset the end coordinate using the actual length of the read. fMax = fMin + FI->fragmentLength(fragId); // Orientation is straightforward, based on the orient of the parent, and the flipped flag. bool fFwd = (((pFwd == true) && (edge->frag3p() == false)) || // parent is fwd, olap is not flipped ((pFwd == false) && (edge->frag3p() == true))); // parent is rev, olap is flipped ufNode frag; frag.ident = fragId; frag.contained = 0; frag.parent = edge->fragId(); // == parent->ident frag.ahang = 0; // Not used in bogart, set on output frag.bhang = 0; // Not used in bogart, set on output frag.position.bgn = (fFwd) ? fMin : fMax; frag.position.end = (fFwd) ? fMax : fMin; #ifdef DEBUG_PLACE_FRAG writeLog("placeCont()-- parent %7d pos %7d,%7d -- edge to %7d %c' hangs %7d %7d -- frag %7d C' -- placed %7d-%7d oriented %s %7d-%7d\n", parent.ident, parent.position.bgn, parent.position.end, edge->fragId(), (edge->frag3p()) ? '3' : '5', edge->ahang(), edge->bhang(), fragId, fMin, fMax, (fFwd) ? "rev" : "fwd", frag.position.bgn, frag.position.end); #endif return(frag); } ufNode placeFrag_dovetail(uint32 fragId, bool frag3p, ufNode &parent, BestEdgeOverlap *edge) { // We have an 'edge' from 'fragId' end 'frag3p' back to 'parent'. // Use that to compute the placement of 'frag'. bool pFwd = (parent.position.bgn < parent.position.end) ? true : false; int32 pMin = (parent.position.bgn < parent.position.end) ? parent.position.bgn : parent.position.end; int32 pMax = (parent.position.bgn < parent.position.end) ? parent.position.end : parent.position.bgn; assert(pMin < pMax); // Scale the hangs based on the placed versus actual length of the parent read. //double intraScale = (double)(pMax - pMin) / FI->fragmentLength(parent.ident); // Within the parent read overlap //double interScale = 1.0; // Outside the parent read overlap // We're given an edge from the read-to-place back to the parent. Reverse the edge so it points // from the parent to the read-to-place. // // The canonical edge is from a forward parent to the child. // // -P----\--> +b // +a ---v--------C- // // To reverse the edge: // // If child is forward, swapping the order of the reads results in a canonical overlap. The // hangs become negative. // // -P----\--> +b ----> -a ---/--------C> // +a ---v--------C> ----> -P----v--> -b // // If child is reverse, swapping the order of the reads results in a backwards canonical // overlap, and we need to flip end-to-end also. The hangs are swapped. // // -P----\--> +b ----> -C--------\--> +a // +a <--v--------C- ----> +b <--v----P- // int32 ahang = (frag3p == false) ? -edge->ahang() : edge->bhang(); int32 bhang = (frag3p == false) ? -edge->bhang() : edge->ahang(); // The read is placed 'to the right' of the parent if // pFwd == true and edge points to 3' end // pFwd == false and edge points to 5' end // bool toRight = (pFwd == edge->frag3p()); // If placing 'to the right', we add hangs. Else, subtract the swapped hangs. int32 fMin = 0; int32 fMax = 0; if (toRight) { fMin = pMin + ahang; fMax = pMax + bhang; } else { fMin = pMin - bhang; fMax = pMax - ahang; } //int32 fMin = pMin + ((frag3p == false) ? -edge->ahang() : edge->bhang()); // * intraScale //int32 fMax = pMax + ((frag3p == false) ? -edge->bhang() : edge->ahang()); // * interScale assert(fMin < fMax); // We don't know the true length of the overlap, and our hang-based math tends to shrink reads. // Reset the end coordinate using the actual length of the read. fMax = fMin + FI->fragmentLength(fragId); // Orientation is a bit more complicated, with eight cases (drawing pictures helps). // // edge from frag3p=true to forward parent 3p -> reverse // edge from frag3p=false to reverse parent 3p -> reverse // edge from frag3p=false to forward parent 5p -> reverse // edge from frag3p=true to reverse parent 5p -> reverse // // edge from frag3p=true to reverse parent 3p -> forward // edge from frag3p=false to forward parent 3p -> forward // edge from frag3p=false to reverse parent 5p -> forward // edge from frag3p=true to forward parent 5p -> forward // bool fFwd = (((frag3p == true) && (pFwd == true) && (edge->frag3p() == true)) || ((frag3p == false) && (pFwd == false) && (edge->frag3p() == true)) || ((frag3p == false) && (pFwd == true) && (edge->frag3p() == false)) || ((frag3p == true) && (pFwd == false) && (edge->frag3p() == false))) ? false : true; ufNode frag; frag.ident = fragId; frag.contained = 0; frag.parent = edge->fragId(); // == parent->ident frag.ahang = 0; // Not used in bogart, set on output frag.bhang = 0; // Not used in bogart, set on output frag.position.bgn = (fFwd) ? fMin : fMax; frag.position.end = (fFwd) ? fMax : fMin; #ifdef DEBUG_PLACE_FRAG writeLog("placeDove()-- parent %7d pos %7d,%7d -- edge to %7d %c' hangs %7d %7d -- frag %7d %c' -- placed %7d-%7d oriented %s %7d-%7d\n", parent.ident, parent.position.bgn, parent.position.end, edge->fragId(), (edge->frag3p()) ? '3' : '5', edge->ahang(), edge->bhang(), fragId, (frag3p) ? '3' : '5', fMin, fMax, (fFwd) ? "rev" : "fwd", frag.position.bgn, frag.position.end); #endif return(frag); } // Place a read into this tig using an edge from the read to some read in this tig. // bool Unitig::placeFrag(ufNode &frag, // output placement uint32 fragId, // id of read we want to place bool frag3p, // end of read 'edge' is from, meaningless if contained BestEdgeOverlap *edge) { // edge to read in this tig assert(fragId > 0); assert(fragId <= FI->numFragments()); frag.ident = fragId; frag.contained = 0; frag.parent = 0; frag.ahang = 0; frag.bhang = 0; frag.position.bgn = 0; frag.position.end = 0; if (edge == NULL) // No best edge? Hard to place without one. return(false); if (edge->fragId() == 0) // Empty best edge? Still hard to place. return(false); if (fragIn(edge->fragId()) != id()) // Edge not pointing to a read in this tig? return(false); // Grab the index of the parent read. uint32 bidx = pathPosition(edge->fragId()); assert(edge->fragId() == ufpath[bidx].ident); // Now, just compute the placement and return success! if (((edge->ahang() >= 0) && (edge->bhang() <= 0)) || ((edge->ahang() <= 0) && (edge->bhang() >= 0))) frag = placeFrag_contained(fragId, ufpath[bidx], edge); else frag = placeFrag_dovetail(fragId, frag3p, ufpath[bidx], edge); return(true); } canu-1.3/src/bogart/addReadsToUnitigs.C000066400000000000000000000374721272335422300200510ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/addReadsToUnitigs.C * * Modifications by: * * Brian P. Walenz from 2013-AUG-24 to 2014-APR-22 * are Copyright 2013-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-09 to 2015-APR-10 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "splitToWords.H" #include "MultiAlign.H" #include "MultiAlignStore.H" #include "MultiAlignment_CNS.H" #include "MultiAlignment_CNS_private.H" #include #include #include using namespace std; class readMap { public: readMap() { good = false; proc = false; rFWD = false; rIID = UINT32_MAX; rCNT = 0; tIID = UINT32_MAX; tBGN = 0; tEND = 0; }; bool good; bool proc; bool rFWD; uint32 rIID; uint32 rCNT; uint32 tIID; uint32 tBGN; uint32 tEND; }; class ungapToGap { public: uint32 *gapToUngap; }; int main(int argc, char **argv) { char *gkpName = NULL; char *tigName = NULL; int32 tigVers = -1; vector alignMapNames; bool doConsensus = false; bool doModify = true; #ifdef UNFINISHED_ADD_TO_SINGLETON bool doPlaceUnmapped = true; bool doDeleteUnmapped = false; #endif int32 numFailures = 0; int32 numSkipped = 0; bool showResult = false; char *lookupFile = NULL; map lookupIID; #ifdef UNFINISHED_ADD_TO_SINGLETON vector iidInTig; // true if the read is already in a unitig vector iidInTigByLib; // count of reads in tigs, by library vector iidInLib; // count of reads, by library #endif bool loadall = false; CNS_Options options = { CNS_OPTIONS_SPLIT_ALLELES_DEFAULT, CNS_OPTIONS_MIN_ANCHOR_DEFAULT, CNS_OPTIONS_DO_PHASING_DEFAULT }; vector RM; vector UG; // Comminucate to MultiAlignment_CNS.c that we are doing consensus and not cgw. thisIsConsensus = 1; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-g") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-t") == 0) { tigName = argv[++arg]; tigVers = atoi(argv[++arg]); if (tigVers <= 0) fprintf(stderr, "invalid tigStore version (-t store version) '-t %s %s'.\n", argv[arg-1], argv[arg]), exit(1); } else if (strcmp(argv[arg], "-m") == 0) { while ((argv[arg+1] != NULL) && (AS_UTL_fileExists(argv[arg+1], false, false) == true)) alignMapNames.push_back(argv[++arg]); } else if (strcmp(argv[arg], "-lookup") == 0) { lookupFile = argv[++arg]; } else if (strcmp(argv[arg], "-r") == 0) { doConsensus = true; } else if (strcmp(argv[arg], "-v") == 0) { showResult = true; } else if (strcmp(argv[arg], "-V") == 0) { VERBOSE_MULTIALIGN_OUTPUT++; } else if (strcmp(argv[arg], "-loadall") == 0) { loadall = true; } else if (strcmp(argv[arg], "-n") == 0) { doModify = false; } else { err++; } arg++; } if (gkpName == NULL) err++; if (tigName == NULL) err++; if (alignMapNames.size() == 0) err++; if (lookupFile == NULL) err++; if (err) { fprintf(stderr, "usage: %s -g gkpStore -t tigStore version -m coords\n", argv[0]); fprintf(stderr, " -g gkpStore gatekeeper store\n"); fprintf(stderr, " -t tigStore version tigStore and version to modify\n"); fprintf(stderr, "\n"); fprintf(stderr, " -m map-file input map coords\n"); fprintf(stderr, " -M fastqUIDmap gatekeeper output fastqUIDmap for read name to IID translation\n"); fprintf(stderr, "\n"); #if 0 fprintf(stderr, "unmapped reads: default is to promote to singleton unitigs\n"); fprintf(stderr, " -U leave unmapped reads alone (will crash CGW)\n"); fprintf(stderr, " -D delete unmapped reads from gkpStore\n"); #else fprintf(stderr, "unmapped reads: all reads that are mapped and eligible for addition must be\n"); fprintf(stderr, "marked as deleted before running this program. reads that are added will be\n"); fprintf(stderr, "undeleted. reads that are not added will remain deleted.\n"); #endif fprintf(stderr, "\n"); fprintf(stderr, "consensus: default is to not rebuild consensus\n"); fprintf(stderr, " -r rebuild consensus including the new reads\n"); fprintf(stderr, " -v show result\n"); fprintf(stderr, " -V verbose\n"); fprintf(stderr, " -loadall load all reads in gkpStore into memory (faster consensus)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -n do all the work, but discard the result\n"); if (gkpName == NULL) fprintf(stderr, "ERROR: no gkpStore (-g) supplied.\n"); if (tigName == NULL) fprintf(stderr, "ERROR: no tigStore (-t) supplied.\n"); if (alignMapNames.size() == 0) fprintf(stderr, "ERROR: no map-file (-m) inputs supplied.\n"); if (lookupFile == NULL) fprintf(stderr, "ERROR: no fasqUIDmap (-M) supplied.\n"); exit(1); } { fprintf(stderr, "Loading Name to IID map from '%s'\n", lookupFile); errno = 0; FILE *LF = fopen(lookupFile, "r"); if (errno) fprintf(stderr, "Failed to open fastqUIDmap '%s'\n", lookupFile); char LL[1024]; fgets(LL, 1024, LF); while (!feof(LF)) { chomp(LL); // Shouldn't be necessary, but splitToWords isn't doing it! splitToWords SW(LL); if (SW.numWords() == 3) { lookupIID[string(SW[2])] = SW(1); } else if (SW.numWords() == 6) { lookupIID[string(SW[2])] = SW(1); lookupIID[string(SW[5])] = SW(4); //fprintf(stderr, "'%s' - %u -- '%s' - %u\n", // SW[2], SW(1), SW[5], SW(4)); } else { } fgets(LL, 1024, LF); } fprintf(stderr, "Loaded "F_SIZE_T" name to IIDs\n", lookupIID.size()); } #ifdef UNFINISHED_ADD_TO_SINGLETON { fprintf(stderr, "Loading tig placement from tigStore '%s'\n", tigName); tigStore = new MultiAlignStore(tigName, tigVers, 0, 0, false, false, false); // Read only for (uint32 ti=0; tinumUnitigs(); ti++) { if (tigStore->isDeleted(ti, true)) continue; MultiAlignT *ma = tigStore->loadMultiAlign(ti, true); if (ma == NULL) continue; uint32 fiMax = GetNumIntMultiPoss(ma->f_list); for (uint32 fi=0; fif_list, fi); iidInTig[imp->ident] = true; } tigStore->unloadMultiAlign(ti, true); } delete tigStore; } #endif // // Load the alignment map. The challenge here is to parse the unitig and read names // into correct IIDs. We assume that: // Reads were dumped with -dumpfasta and have names ">UID,IID" // Unitigs were dumped with tigStore -d consensus and have names "utgIID" // Alignments are in the convertToExtent -extended format // uint32 totAligns = 0; uint32 totUnmap = 0; uint32 totUnique = 0; uint32 totDups = 0; uint32 lowID = UINT32_MAX; uint32 maxID = 0; for (uint32 an=0; an 1) totDups++; } fprintf(stderr, "Loaded %u aligns from ID %u to %u, %u reads unmapped, %u reads had a unique alignment and %u reads had multiple aligns.\n", totAligns, lowID, maxID, totUnmap, totUnique, totDups); // // Update deletion status in gkpStore. This processes every read, regardless. // fprintf(stderr, "Processing mate pairs, updating gkpStore.\n"); gkpStore = gkStore::gkStore_open(gkpName, false, true); // last arg - TRUE - writable uint32 unpaired = 0; uint32 multiple = 0; uint32 pairsToSame = 0; uint32 pairsToDiff = 0; #ifdef UNFINISHED_ADD_TO_SINGLETON // Need to process all reads, since we don't know where the first/last unmapped read is! // We could instead process from the first to last deleted read in gkpStore, or ask which // libraries were being added. lowID = 1; maxID = gkpStore->gkStore_getNumFragments(); #endif for (uint32 ff=lowID, mm=lowID; ff<=maxID; ff++) { gkFragment read; gkFragment mate; // I think this is just a short circuit of the two checks of 'one or both reads has too // few/many mappings' below. //if (RM[ff].rCNT != 1) // // Not mapped, mapped too much // continue; gkpStore->gkStore_getFragment(ff, &read, GKFRAGMENT_INF); mm = read.gkFragment_getMateIID(); if (mm == 0) // No mate, pacbio read? continue; if (mm < ff) // Already processed. continue; gkpStore->gkStore_getFragment(mm, &mate, GKFRAGMENT_INF); if ((RM[ff].rCNT == 0) || (RM[mm].rCNT == 0)) { // One or both reads has too few mappings. unpaired++; RM[ff].good = false; RM[mm].good = false; continue; } if ((RM[ff].rCNT > 1) || (RM[mm].rCNT > 1)) { // One or both reads has too many mappings. multiple++; RM[ff].good = false; RM[mm].good = false; continue; } RM[ff].good = true; RM[mm].good = true; read.gkFragment_setIsDeleted(false); mate.gkFragment_setIsDeleted(false); gkpStore->gkStore_setFragment(&read); gkpStore->gkStore_setFragment(&mate); if (RM[ff].tIID == RM[mm].tIID) pairsToSame++; else pairsToDiff++; } gkpStore->gkStore_close(); fprintf(stderr, "Will NOT add %u pairs - one read failed to map.\n", unpaired); fprintf(stderr, "Will NOT add %u pairs - multiple mappings.\n", multiple); fprintf(stderr, "Will add %u pairs in the same unitig\n", pairsToSame); fprintf(stderr, "Will add %u pairs in different unitigs\n", pairsToDiff); // // Open stores. gkpStore cannot be opened for writing, because then we can't loadall. // gkpStore = gkStore::gkStore_open(gkpName, false, false); // last arg - false - not writable tigStore = new MultiAlignStore(tigName, tigVers, 0, 0, true, true, false); // Write back to the same version if (loadall) { fprintf(stderr, "Loading all reads into memory.\n"); gkpStore->gkStore_load(0, 0, GKFRAGMENT_QLT); // fails if gkStore is writable } // // Rebuild unitigs, stuff them back into the same version. // // Argh, really should convert this to a vector right now.... VA_TYPE(int32) *unGappedOffsets = CreateVA_int32(1024 * 1024); for (uint32 bb=0; bbloadMultiAlign(RM[bb].tIID, true); uint32 ungapLength = GetMultiAlignUngappedLength(ma); uint32 gapLength = GetMultiAlignLength(ma); vector ungapToGap; GetMultiAlignUngapToGap(ma, ungapToGap); //fprintf(stderr, "Loaded UTG %u offset size %lu\n", ma->maID, ungapToGap.size()); //for (uint32 xx=0; xx gap %u\n", xx, ungapToGap[xx]); uint32 readsAdded = 0; for (uint32 ee=bb; eemaID) || (RM[ee].good == false)) continue; uint32 bgn = ungapToGap[RM[ee].tBGN]; uint32 end = ungapToGap[RM[ee].tEND]; readsAdded++; fprintf(stdout, "bb=%u ee=%u ADD frag %u to unitig %u at %u,%u (from ungapped %u,%u)\n", bb, ee, RM[ee].rIID, RM[ee].tIID, bgn, end, RM[ee].tBGN, RM[ee].tEND); // Add a read to the unitig. IntMultiPos frg; frg.type = AS_READ; frg.ident = RM[ee].rIID; frg.contained = 0; frg.parent = 0; frg.ahang = 0; frg.bhang = 0; frg.position.bgn = (RM[ee].rFWD) ? bgn : end, frg.position.end = (RM[ee].rFWD) ? end : bgn, frg.delta_length = 0; frg.delta = NULL; AppendVA_IntMultiPos(ma->f_list, &frg); // Mark that we've processed this read. assert(RM[ee].proc == false); RM[ee].proc = true; } fprintf(stderr, "Added %u reads to unitig %u (previously %lu reads)\n", readsAdded, ma->maID, GetNumIntMultiPoss(ma->f_list) - readsAdded); if (doConsensus) { fprintf(stderr, "Regenerating consensus.\n"); if (MultiAlignUnitig(ma, gkpStore, &options, NULL)) { if (showResult) PrintMultiAlignT(stdout, ma, gkpStore, false, false, AS_READ_CLEAR_LATEST); } else { fprintf(stderr, "MultiAlignUnitig()-- unitig %d failed.\n", ma->maID); numFailures++; } } if (doModify) { fprintf(stderr, "Updating unitig %u\n", ma->maID); tigStore->insertMultiAlign(ma, true, false); } } delete gkpStore; delete tigStore; exit(0); } canu-1.3/src/bogart/analyzeBest.C000066400000000000000000000155701272335422300167500ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BOG/analyzeBest.C * * Modifications by: * * Brian P. Walenz from 2010-OCT-09 to 2013-AUG-01 * are Copyright 2010-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-09 to 2015-APR-10 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "AS_PER_gkpStore.H" #include "splitToWords.H" // Read the 'best.edges' and 'best.contains' outputs from BOG, compute // how many singletons, spurs and contains are present per library. // // Assumes it is run from 4-unitigger. int main(int argc, char **argv) { char *gkpName = 0L; char *bEdge = "best.edges"; char *bCont = "best.contains"; char *bSing = "best.singletons"; int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-g") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-e") == 0) { bEdge = argv[++arg]; } else if (strcmp(argv[arg], "-c") == 0) { bCont = argv[++arg]; } else if (strcmp(argv[arg], "-s") == 0) { bSing = argv[++arg]; } else { err++; } arg++; } if ((err) || (gkpName == 0L)) { fprintf(stderr, "usage: %s -g gkpName [-e best.edges] [-c best.contains]\n", argv[0]); exit(1); } fprintf(stderr, "Opening best edges and contains.\n"); errno = 0; FILE *be = fopen(bEdge, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", bEdge, strerror(errno)), exit(1); FILE *bc = fopen(bCont, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", bCont, strerror(errno)), exit(1); FILE *bs = fopen(bSing, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", bSing, strerror(errno)), exit(1); fprintf(stderr, "Loading fragment to library mapping.\n"); gkStore *gkp = gkStore::gkStore_open(gkpName, false, false); gkStream *str = new gkStream(gkp, 0, 0, GKFRAGMENT_INF); gkFragment fr; uint32 numFrg = gkp->gkStore_getNumFragments(); uint32 numLib = gkp->gkStore_getNumLibraries(); uint32 *frgToLib = new uint32 [numFrg + 1]; memset(frgToLib, 0, sizeof(uint32) * (numFrg + 1)); uint64 *fragPerLib = new uint64 [numLib + 1]; memset(fragPerLib, 0, sizeof(uint64) * (numLib + 1)); uint64 *deldPerLib = new uint64 [numLib + 1]; memset(deldPerLib, 0, sizeof(uint64) * (numLib + 1)); while (str->next(&fr)) { frgToLib[fr.gkFragment_getReadIID()] = fr.gkFragment_getLibraryIID(); if (fr.gkFragment_getIsDeleted()) deldPerLib[fr.gkFragment_getLibraryIID()]++; else fragPerLib[fr.gkFragment_getLibraryIID()]++; } delete str; uint64 *cntdPerLib = new uint64 [numLib + 1]; memset(cntdPerLib, 0, sizeof(uint64) * (numLib + 1)); uint64 *cntrPerLib = new uint64 [numLib + 1]; memset(cntrPerLib, 0, sizeof(uint64) * (numLib + 1)); uint32 *cntr = new uint32 [numFrg + 1]; memset(cntr, 0, sizeof(uint32) * (numFrg + 1)); uint64 *singPerLib = new uint64 [numLib + 1]; memset(singPerLib, 0, sizeof(uint64) * (numLib + 1)); uint64 *spu5PerLib = new uint64 [numLib + 1]; memset(spu5PerLib, 0, sizeof(uint64) * (numLib + 1)); uint64 *spu3PerLib = new uint64 [numLib + 1]; memset(spu3PerLib, 0, sizeof(uint64) * (numLib + 1)); uint64 *dovePerLib = new uint64 [numLib + 1]; memset(dovePerLib, 0, sizeof(uint64) * (numLib + 1)); int32 lineMax = 1048576; char *line = new char [lineMax]; splitToWords W; fprintf(stderr, "Processing best.singletons.\n"); while (!feof(bs)) { fgets(line, lineMax, bs); chomp(line); W.split(line); if (line[0] == '#') continue; singPerLib[frgToLib[W(0)]]++; } fprintf(stderr, "Processing best.contains.\n"); while (!feof(bc)) { fgets(line, lineMax, bc); chomp(line); W.split(line); if (line[0] == '#') continue; cntdPerLib[frgToLib[W(0)]]++; cntr[W(3)]++; } for (uint32 i=1; i<=numFrg; i++) { if (cntr[i] > 0) cntrPerLib[frgToLib[i]]++; } fprintf(stderr, "Processing best.edges.\n"); while (!feof(be)) { fgets(line, lineMax, be); chomp(line); W.split(line); if (line[0] == '#') continue; if ((W(2) == 0) && (W(4) == 0)) singPerLib[frgToLib[W(0)]]++; else if (W(2) == 0) spu5PerLib[frgToLib[W(0)]]++; else if (W(4) == 0) spu3PerLib[frgToLib[W(0)]]++; else dovePerLib[frgToLib[W(0)]]++; } fprintf(stderr, "libIID libUID #frg #del cnt'd cnt'r sing spur5 spur3 dove\n"); for (uint32 i=0; igkStore_getLibrary(i)->libraryName, fragPerLib[i], deldPerLib[i], (deldPerLib[i] == 0) ? 0.0 : 100.0 * deldPerLib[i] / tot, cntdPerLib[i], (cntdPerLib[i] == 0) ? 0.0 : 100.0 * cntdPerLib[i] / tot, cntrPerLib[i], (cntrPerLib[i] == 0) ? 0.0 : 100.0 * cntrPerLib[i] / tot, singPerLib[i], (singPerLib[i] == 0) ? 0.0 : 100.0 * singPerLib[i] / tot, spu5PerLib[i], (spu5PerLib[i] == 0) ? 0.0 : 100.0 * spu5PerLib[i] / tot, spu3PerLib[i], (spu3PerLib[i] == 0) ? 0.0 : 100.0 * spu3PerLib[i] / tot, dovePerLib[i], (dovePerLib[i] == 0) ? 0.0 : 100.0 * dovePerLib[i] / tot); } gkp->gkStore_close(); fclose(be); fclose(bc); delete [] frgToLib; delete [] fragPerLib; delete [] deldPerLib; delete [] cntdPerLib; delete [] cntrPerLib; delete [] cntr; delete [] singPerLib; delete [] spu5PerLib; delete [] spu3PerLib; delete [] dovePerLib; delete [] line; return(0); } canu-1.3/src/bogart/bogart.C000066400000000000000000000442501272335422300157420ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/bogart.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2014-JAN-29 * are Copyright 2010-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-21 to 2015-AUG-07 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-MAR-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_BAT_FragmentInfo.H" #include "AS_BAT_OverlapCache.H" #include "AS_BAT_BestOverlapGraph.H" #include "AS_BAT_ChunkGraph.H" #include "AS_BAT_Logging.H" #include "AS_BAT_Unitig.H" #include "AS_BAT_PopulateUnitig.H" #include "AS_BAT_Instrumentation.H" #include "AS_BAT_PlaceContains.H" #include "AS_BAT_MergeUnitigs.H" #include "AS_BAT_PopBubbles.H" #include "AS_BAT_MarkRepeatReads.H" #include "AS_BAT_SplitDiscontinuous.H" #include "AS_BAT_PromoteToSingleton.H" #include "AS_BAT_SetParentAndHang.H" #include "AS_BAT_Outputs.H" FragmentInfo *FI = 0L; OverlapCache *OC = 0L; BestOverlapGraph *OG = 0L; ChunkGraph *CG = 0L; int main (int argc, char * argv []) { char *gkpStorePath = NULL; char *ovlStoreUniqPath = NULL; char *ovlStoreReptPath = NULL; char *tigStorePath = NULL; double erateGraph = 0.030; double erateMax = 0.050; uint64 genomeSize = 0; uint32 fewReadsNumber = 2; // Parameters for labeling of unassembled; also set in pipelines/canu/Defaults.pm uint32 tooShortLength = 1000; double spanFraction = 0.75; double lowcovFraction = 0.75; uint32 lowcovDepth = 2; double deviationGraph = 5.0; double deviationBubble = 5.0; double deviationRepeat = 3.0; uint32 confusedAbsolute = 250; double confusedPercent = 100.0; int32 numThreads = 0; uint64 ovlCacheMemory = UINT64_MAX; uint32 ovlCacheLimit = UINT32_MAX; bool onlySave = false; bool doSave = false; int fragment_count_target = 0; char *prefix = NULL; bool enableJoining = false; bool enableShatterRepeats = false; bool enableReconstructRepeats = false; uint32 minReadLen = 0; uint32 minOverlap = 500; argc = AS_configure(argc, argv); vector err; int arg = 1; while (arg < argc) { if (strcmp(argv[arg], "-B") == 0) { fragment_count_target = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-o") == 0) { prefix = argv[++arg]; } else if (strcmp(argv[arg], "-G") == 0) { gkpStorePath = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { if (ovlStoreUniqPath == NULL) ovlStoreUniqPath = argv[++arg]; else if (ovlStoreReptPath == NULL) ovlStoreReptPath = argv[++arg]; else err.push_back(NULL); } else if (strcmp(argv[arg], "-gs") == 0) { genomeSize = strtoull(argv[++arg], NULL, 10); } else if (strcmp(argv[arg], "-J") == 0) { enableJoining = true; } else if (strcmp(argv[arg], "-T") == 0) { tigStorePath = argv[++arg]; } else if (strcmp(argv[arg], "-SR") == 0) { enableShatterRepeats = true; } else if (strcmp(argv[arg], "-R") == 0) { enableShatterRepeats = true; enableReconstructRepeats = true; } else if (strcmp(argv[arg], "-unassembled") == 0) { fewReadsNumber = atoi(argv[++arg]); tooShortLength = atoi(argv[++arg]); spanFraction = atof(argv[++arg]); lowcovFraction = atof(argv[++arg]); lowcovDepth = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-RL") == 0) { minReadLen = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-threads") == 0) { numThreads = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-eg") == 0) { erateGraph = atof(argv[++arg]); } else if (strcmp(argv[arg], "-eM") == 0) { erateMax = atof(argv[++arg]); } else if (strcmp(argv[arg], "-el") == 0) { minOverlap = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-ca") == 0) { // Edge confused, based on absolute difference confusedAbsolute = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-cp") == 0) { // Edge confused, based on percent difference confusedPercent = atof(argv[++arg]); } else if (strcmp(argv[arg], "-dg") == 0) { // Deviations, graph deviationGraph = atof(argv[++arg]); } else if (strcmp(argv[arg], "-db") == 0) { // Deviations, bubble deviationBubble = atof(argv[++arg]); } else if (strcmp(argv[arg], "-dr") == 0) { // Deviations, repeat deviationRepeat = atof(argv[++arg]); } else if (strcmp(argv[arg], "-M") == 0) { ovlCacheMemory = (uint64)(atof(argv[++arg]) * 1024 * 1024 * 1024); } else if (strcmp(argv[arg], "-N") == 0) { ovlCacheLimit = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-create") == 0) { onlySave = true; doSave = true; } else if (strcmp(argv[arg], "-save") == 0) { doSave = true; } else if (strcmp(argv[arg], "-D") == 0) { uint32 opt = 0; uint64 flg = 1; bool fnd = false; for (arg++; logFileFlagNames[opt]; flg <<= 1, opt++) { if (strcasecmp(logFileFlagNames[opt], argv[arg]) == 0) { logFileFlags |= flg; fnd = true; } } if (strcasecmp("all", argv[arg]) == 0) { for (flg=1, opt=0; logFileFlagNames[opt]; flg <<= 1, opt++) if (strcasecmp(logFileFlagNames[opt], "stderr") != 0) logFileFlags |= flg; fnd = true; } if (strcasecmp("most", argv[arg]) == 0) { for (flg=1, opt=0; logFileFlagNames[opt]; flg <<= 1, opt++) if ((strcasecmp(logFileFlagNames[opt], "stderr") != 0) && (strcasecmp(logFileFlagNames[opt], "overlapScoring") != 0) && (strcasecmp(logFileFlagNames[opt], "chunkGraph") != 0) && (strcasecmp(logFileFlagNames[opt], "setParentAndHang") != 0)) logFileFlags |= flg; fnd = true; } if (fnd == false) { char *s = new char [1024]; sprintf(s, "Unknown '-D' option '%s'.\n", argv[arg]); err.push_back(s); } } else if (strcmp(argv[arg], "-d") == 0) { uint32 opt = 0; uint64 flg = 1; bool fnd = false; for (arg++; logFileFlagNames[opt]; flg <<= 1, opt++) { if (strcasecmp(logFileFlagNames[opt], argv[arg]) == 0) { logFileFlags &= ~flg; fnd = true; } } if (fnd == false) { char *s = new char [1024]; sprintf(s, "Unknown '-d' option '%s'.\n", argv[arg]); err.push_back(s); } } else { char *s = new char [1024]; sprintf(s, "Unknown option '%s'.\n", argv[arg]); err.push_back(s); } arg++; } if (erateGraph < 0.0) err.push_back(NULL); if (erateMax < 0.0) err.push_back(NULL); if (prefix == NULL) err.push_back(NULL); if (gkpStorePath == NULL) err.push_back(NULL); if (ovlStoreUniqPath == NULL) err.push_back(NULL); if (tigStorePath == NULL) err.push_back(NULL); if (err.size() > 0) { fprintf(stderr, "usage: %s -o outputName -O ovlStore -G gkpStore -T tigStore\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " -O Mandatory path to an ovlStore.\n"); fprintf(stderr, " -G Mandatory path to a gkpStore.\n"); fprintf(stderr, " -T Mandatory path to a tigStore (can exist or not).\n"); fprintf(stderr, " -o prefix Mandatory name for the output files\n"); fprintf(stderr, "\n"); fprintf(stderr, " -B b Target number of fragments per tigStore (consensus) partition\n"); fprintf(stderr, "\n"); fprintf(stderr, "Algorithm Options\n"); fprintf(stderr, "\n"); fprintf(stderr, " -gs Genome size in bases.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -J Join promiscuous unitigs using unused best edges.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -SR Shatter repeats, don't rebuild.\n"); fprintf(stderr, " -R Shatter repeats (-SR), then rebuild them\n"); fprintf(stderr, " -RL len Force reads below 'len' bases to be singletons.\n"); fprintf(stderr, " This WILL cause CGW to fail; diagnostic only.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -threads N Use N compute threads during repeat detection.\n"); fprintf(stderr, " 0 - use OpenMP default (default)\n"); fprintf(stderr, " 1 - use one thread\n"); fprintf(stderr, "\n"); fprintf(stderr, "Overlap Selection - an overlap will be considered for use in a unitig under\n"); fprintf(stderr, " the following conditions:\n"); fprintf(stderr, "\n"); fprintf(stderr, " When constructing the Best Overlap Graph and Promiscuous Unitigs ('g'raph):\n"); fprintf(stderr, " -eg 0.020 no more than 0.020 fraction (2.0%%) error ** DEPRECATED **\n"); fprintf(stderr, "\n"); fprintf(stderr, " When loading overlaps, an inflated maximum (to allow reruns with different error rates):\n"); fprintf(stderr, " -eM 0.05 no more than 0.05 fraction (5.0%%) error in any overlap loaded into bogart\n"); fprintf(stderr, " the maximum used will ALWAYS be at leeast the maximum of the four error rates\n"); fprintf(stderr, "\n"); fprintf(stderr, " For all, the lower limit on overlap length\n"); fprintf(stderr, " -el 500 no shorter than 40 bases\n"); fprintf(stderr, "\n"); fprintf(stderr, "Overlap Storage\n"); fprintf(stderr, "\n"); fprintf(stderr, " -M gb Use at most 'gb' gigabytes of memory for storing overlaps.\n"); fprintf(stderr, " -N num Load at most 'num' overlaps per read.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -create Only create the overlap graph, save to disk and quit.\n"); fprintf(stderr, " -save Save the overlap graph to disk, and continue.\n"); fprintf(stderr, "\n"); fprintf(stderr, "Debugging and Logging\n"); fprintf(stderr, "\n"); fprintf(stderr, " -D enable logging/debugging for a specific component.\n"); fprintf(stderr, " -d disable logging/debugging for a specific component.\n"); for (uint32 l=0; logFileFlagNames[l]; l++) fprintf(stderr, " %s\n", logFileFlagNames[l]); fprintf(stderr, "\n"); if (erateGraph < 0.0) fprintf(stderr, "Invalid overlap error threshold (-eg option); must be at least 0.0.\n"); if (erateMax < 0.0) fprintf(stderr, "Invalid overlap error threshold (-eM option); must be at least 0.0.\n"); if (prefix == NULL) fprintf(stderr, "No output prefix name (-o option) supplied.\n"); if (gkpStorePath == NULL) fprintf(stderr, "No gatekeeper store (-G option) supplied.\n"); if (ovlStoreUniqPath == NULL) fprintf(stderr, "No overlap store (-O option) supplied.\n"); if ((ovlStoreUniqPath != NULL) && (ovlStoreUniqPath == ovlStoreReptPath)) fprintf(stderr, "Too many overlap stores (-O option) supplied.\n"); if (tigStorePath == NULL) fprintf(stderr, "No output tigStore (-T option) supplied.\n"); for (uint32 ii=0; ii 0) { omp_set_num_threads(numThreads); fprintf(stderr, "number of threads = %d (command line)\n", numThreads); fprintf(stderr, "\n"); } else { fprintf(stderr, "number of threads = %d (OpenMP default)\n", omp_get_max_threads()); fprintf(stderr, "\n"); } for (uint64 i=0, j=1; i<64; i++, j<<=1) if (logFileFlagSet(j)) fprintf(stderr, "DEBUG = %s\n", logFileFlagNames[i]); gkStore *gkpStore = gkStore::gkStore_open(gkpStorePath); ovStore *ovlStoreUniq = new ovStore(ovlStoreUniqPath, gkpStore); ovStore *ovlStoreRept = ovlStoreReptPath ? new ovStore(ovlStoreReptPath, gkpStore) : NULL; UnitigVector unitigs; setLogFile(prefix, NULL); FI = new FragmentInfo(gkpStore, prefix, minReadLen); // Initialize where we've been to nowhere Unitig::resetFragUnitigMap(FI->numFragments()); OC = new OverlapCache(ovlStoreUniq, ovlStoreRept, prefix, MAX(erateMax, erateGraph), minOverlap, ovlCacheMemory, ovlCacheLimit, onlySave, doSave); OG = new BestOverlapGraph(erateGraph, deviationGraph, prefix); CG = new ChunkGraph(prefix); delete ovlStoreUniq; ovlStoreUniq = NULL; delete ovlStoreRept; ovlStoreRept = NULL; gkpStore->gkStore_close(); gkpStore = NULL; // // Build the initial unitig path from non-contained fragments. The first pass is usually the // only one needed, but occasionally (maybe) we miss fragments, so we make an explicit pass // through all fragments and place whatever isn't already placed. // setLogFile(prefix, "buildUnitigs"); writeLog("==> BUILDING UNITIGS from %d fragments.\n", FI->numFragments()); for (uint32 fi=CG->nextFragByChunkLength(); fi>0; fi=CG->nextFragByChunkLength()) populateUnitig(unitigs, fi); delete CG; CG = NULL; breakSingletonTigs(unitigs); reportOverlaps(unitigs, prefix, "buildUnitigs"); reportUnitigs(unitigs, prefix, "buildUnitigs", genomeSize); // // Place contained reads. // #if 1 setLogFile(prefix, "placeContains"); unitigs.computeArrivalRate(prefix, "initial"); unitigs.computeErrorProfiles(prefix, "initial"); //unitigs.reportErrorProfiles(prefix, "initial"); placeUnplacedUsingAllOverlaps(unitigs, prefix); reportOverlaps(unitigs, prefix, "placeContains"); reportUnitigs(unitigs, prefix, "placeContains", genomeSize); #endif // // Merge tigs (and detect ciruclar ones too). Contained reads need to be placed to 'clean up' // the error rate. Dovetail alone is too 'clean' for circular to be detected (in ecoli). // #if 0 setLogFile(prefix, "merge"); computeErrorProfiles(unitigs, prefix, "merge"); //reportErrorProfiles(unitigs, prefix, "merge"); mergeUnitigs(unitigs, deviationGraph, false); reportOverlaps(unitigs, prefix, "merge"); reportUnitigs(unitigs, prefix, "merge", genomeSize); #endif // // Pop bubbles // #if 1 setLogFile(prefix, "popBubbles"); unitigs.computeErrorProfiles(prefix, "unplaced"); //unitigs.reportErrorProfiles(prefix, "unplaced"); popBubbles(unitigs, deviationBubble); //checkUnitigMembership(unitigs); reportOverlaps(unitigs, prefix, "popBubbles"); reportUnitigs(unitigs, prefix, "popBubbles", genomeSize); #endif // // Detect and break repeats. Annotate each read with overlaps to reads not overlapping in the tig, // project these regions back to the tig, and break unless there is a read spanning the region. // setLogFile(prefix, "markRepeatReads"); unitigs.computeErrorProfiles(prefix, "repeats"); markRepeatReads(unitigs, deviationRepeat, confusedAbsolute, confusedPercent); //checkUnitigMembership(unitigs); reportOverlaps(unitigs, prefix, "markRepeatReads"); reportUnitigs(unitigs, prefix, "markRepeatReads", genomeSize); // // Try to reassemble just the split repeats. // #if 0 if (enableReconstructRepeats) { assert(enableShatterRepeats); setLogFile(prefix, "reconstructRepeats"); reconstructRepeats(unitigs, erateGraph); //checkUnitigMembership(unitigs); reportOverlaps(unitigs, prefix, "reconstructRepeats"); reportUnitigs(unitigs, prefix, "reconstructRepeats", genomeSize); } #endif // // Cleanup unitigs. Break those that have gaps in them. Place contains again. For any read // still unplaced, make it a singleton unitig. // setLogFile(prefix, "cleanup"); splitDiscontinuousUnitigs(unitigs, minOverlap); breakSingletonTigs(unitigs); //unitigs.computeErrorProfiles(prefix, "final"); //unitigs.reportErrorProfiles(prefix, "final"); //placeUnplacedUsingAllOverlaps(unitigs, prefix); promoteToSingleton(unitigs); classifyUnitigsAsUnassembled(unitigs, fewReadsNumber, tooShortLength, spanFraction, lowcovFraction, lowcovDepth); //checkUnitigMembership(unitigs); reportOverlaps(unitigs, prefix, "final"); reportUnitigs(unitigs, prefix, "final", genomeSize); // // Generate outputs. The graph MUST come after output, because it needs // the tigStore tigID. // setLogFile(prefix, "output"); setParentAndHang(unitigs); writeUnitigsToStore(unitigs, prefix, tigStorePath, fragment_count_target, true); setLogFile(prefix, "graph"); writeUnusedEdges(unitigs, prefix); // // Tear down bogart. // delete CG; delete OG; delete OC; delete FI; setLogFile(prefix, NULL); writeLog("Bye.\n"); return(0); } canu-1.3/src/bogart/bogart.mk000066400000000000000000000023641272335422300161670ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := bogart SOURCES := bogart.C \ AS_BAT_BestOverlapGraph.C \ AS_BAT_ChunkGraph.C \ AS_BAT_FragmentInfo.C \ AS_BAT_Instrumentation.C \ AS_BAT_Logging.C \ AS_BAT_MarkRepeatReads.C \ AS_BAT_MergeUnitigs.C \ AS_BAT_Outputs.C \ AS_BAT_OverlapCache.C \ AS_BAT_PlaceContains.C \ AS_BAT_PlaceFragUsingOverlaps.C \ AS_BAT_PopBubbles.C \ AS_BAT_PopulateUnitig.C \ AS_BAT_PromoteToSingleton.C \ AS_BAT_ReconstructRepeats.C \ AS_BAT_SetParentAndHang.C \ AS_BAT_SplitDiscontinuous.C \ AS_BAT_Unitig.C \ AS_BAT_UnitigVector.C \ AS_BAT_Unitig_AddFrag.C \ AS_BAT_Unitig_PlaceFragUsingEdges.C SRC_INCDIRS := .. ../AS_UTL ../stores TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/bogart/buildGraph.C000066400000000000000000000261371272335422300165510ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-FEB-12 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" #include "splitToWords.H" #include using namespace std; class bestRead { public: bestRead() { }; ~bestRead() { }; uint32 tigID; char tigType; uint32 readID; uint32 readBgn; uint32 readEnd; }; class bestEdge { public: bestEdge() { }; ~bestEdge() { }; bestRead fr; int32 ahang; int32 bhang; bestRead to; bool flipped; }; void loadEdges(char *edgeName, vector &edges, set &vertices) { errno = 0; FILE *edgeFile = fopen(edgeName, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", edgeName, strerror(errno)); char edgeLine[1024]; fgets(edgeLine, 1024, edgeFile); while (!feof(edgeFile)) { splitToWords W(edgeLine); bestEdge E; uint32 w = 0; assert(W[w++][0] == 't'); // 'tig' E.fr.tigID = W(w++); // tigID E.fr.tigType = W[w++][0]; // tig type 'R', 'N', ... assert(W[w++][0] == 'r'); // 'read' E.fr.readID = W(w++); // readID assert(W[w++][0] == 'a'); // 'at' E.fr.readBgn = W(w++); // bgn-position E.fr.readEnd = W(w++); // end-position E.ahang = W(w++); // a-hang E.flipped = (W[w++][0] == '<'); // '<' if flipped, '>' if normal E.bhang = W(w++); // b-hang assert(W[w++][0] == 't'); // 'tig' E.to.tigID = W(w++); // tigID E.fr.tigType = W[w++][0]; // tig type assert(W[w++][0] == 'r'); // 'read' E.to.readID = W(w++); // readID assert(W[w++][0] == 'a'); // 'at' E.to.readBgn = W(w++); // bgn-position E.to.readEnd = W(w++); // end-position edges.push_back(E); vertices.insert(E.fr.tigID); vertices.insert(E.to.tigID); fgets(edgeLine, 1024, edgeFile); } fprintf(stderr, "Loaded "F_SIZE_T" edges from '%s'.\n", edges.size(), edgeName); } tgPosition * findRead(tgTig *tig, uint32 id) { uint32 rr = 0; tgPosition *rd = NULL; do { rd = tig->getChild(rr++); } while ((rr < tig->numberOfChildren()) && (rd->ident() != id)); if (rd->ident() != id) { fprintf(stderr, "WARNING: failed to find read %u in tig %u - ejected?\n", id, tig->tigID()); rd = NULL; } return(rd); } int main(int argc, char **argv) { char *gkpName = NULL; char *tigName = NULL; int32 tigVers = -1; char *edgesName = NULL; char *graphName = NULL; argc = AS_configure(argc, argv); vector err; int arg = 1; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { tigName = argv[++arg]; tigVers = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-E") == 0) { edgesName = argv[++arg]; } else if (strcmp(argv[arg], "-o") == 0) { graphName = argv[++arg]; } else { char *s = new char [1024]; sprintf(s, "Unknown option '%s'.\n", argv[arg]); err.push_back(s); } arg++; } if (gkpName == NULL) err.push_back("No gatekeeper store (-G option) supplied.\n"); if ((tigName == NULL) || (tigVers == -1)) err.push_back("No tigStore store (-T option) supplied.\n"); if (edgesName == NULL) err.push_back("No edges file (-E option) supplied.\n"); if (graphName == NULL) err.push_back("No output graph file (-o option) supplied.\n"); if (err.size() > 0) { fprintf(stderr, "usage: %s -G gkpStore -T tigStore tigVersion -E edgesFile ...\n", argv[0]); fprintf(stderr, " -G gkpStore path to gkpStore\n"); fprintf(stderr, " -T tigStore version path to tigStore\n"); fprintf(stderr, " -E edgeFile path to bogart unused-edges file\n"); fprintf(stderr, "\n"); fprintf(stderr, " -o graph.gfa write to 'graph.gfa'\n"); fprintf(stderr, "\n"); for (uint32 ii=0; ii edges; set vertices; loadEdges(edgesName, edges, vertices); // Dump vertcies. fprintf(graph, "H\tVN:Z:canu\n"); for (uint32 tt=0; ttnumTigs(); tt++) { tgTig *tig = tigStore->loadTig(tt); if (vertices.count(tig->tigID()) > 0) fprintf(graph, "S\ttig%08u\t*\tLN:i:%u\tRC:i:%u\n", tig->tigID(), tig->length(), tig->numberOfChildren()); tigStore->unloadTig(tt, true); } // Dump graph. char *cigar = new char [1024 * 1024]; uint32 nEdgesUnassembled = 0; for (uint32 ee=0; eeloadTig(edges[ee].fr.tigID); uint32 frTigID = frTig->tigID(); tgTig *toTig = tigStore->loadTig(edges[ee].to.tigID); uint32 toTigID = toTig->tigID(); assert(frTig->_class != tgTig_noclass); assert(toTig->_class != tgTig_noclass); if ((frTig->_class == tgTig_unassembled) || (toTig->_class == tgTig_unassembled)) { nEdgesUnassembled++; continue; } // Find the reads we're using to anchor the tigs together. tgPosition *frRead = findRead(frTig, edges[ee].fr.readID); tgPosition *toRead = findRead(toTig, edges[ee].to.readID); if ((frRead == NULL) || (toRead == NULL)) continue; // Map coordinates from gapped to ungapped. uint32 frReadMin = frTig->mapGappedToUngapped(frRead->min()); uint32 frReadMax = frTig->mapGappedToUngapped(frRead->max()); uint32 frLen = frTig->length(false); uint32 toReadMin = toTig->mapGappedToUngapped(toRead->min()); uint32 toReadMax = toTig->mapGappedToUngapped(toRead->max()); uint32 toLen = toTig->length(false); // // Convert from a read-read overlap to a tig-tig overlap. // // Orient tigs based oin the read orientation. // For 'fr', we require that the read always be forward. // For 'to', the overlap dictates the orientation. bool frTigFwd = frRead->isForward(); // Tig forward if read forward. bool toTigFwd = toRead->isForward(); // Same, unless... if (edges[ee].flipped == true) // ...edge is flipped, so flip toTigFwd = !toTigFwd; // 'to' tig. // Cleanup. Makes skipping an edge much easier. tigStore->unloadTig(frTigID, true); frRead = NULL; tigStore->unloadTig(toTigID, true); toRead = NULL; // Based on tig orientation, find the bgn and end lengths from each read. int32 frBgn = (frTigFwd) ? ( frReadMin) : (frLen - frReadMax); int32 frEnd = (frTigFwd) ? (frLen - frReadMax) : ( frReadMin); int32 toBgn = (toTigFwd) ? ( toReadMin) : (toLen - toReadMax); int32 toEnd = (toTigFwd) ? (toLen - toReadMax) : ( toReadMin); //fprintf(graph, "hangs0- fr %d-%d to %d-%d ahang %d bhang %d\n", // frBgn, frEnd, toBgn, toEnd, edges[ee].ahang, edges[ee].bhang); // Apply the overlap hangs to find the overlapping regions on the tigs. if (edges[ee].ahang < 0) toBgn += -edges[ee].ahang; else frBgn += edges[ee].ahang; if (edges[ee].bhang < 0) frEnd += -edges[ee].bhang; else toEnd += edges[ee].bhang; //fprintf(graph, "hangs1- fr %d-%d to %d-%d\n", // frBgn, frEnd, toBgn, toEnd); // The overlap is now between regions toBgn-toEnd and frBgn-frEnd. Extend this to cover the ends of each tig. // // ------------------------------------------ // +++ ||| olap ||| +++ // ------------------------------- if (toBgn < frBgn) { toBgn -= toBgn; frBgn -= toBgn; } else { toBgn -= frBgn; frBgn -= frBgn; } if (toEnd < frEnd) { toEnd -= toEnd; frEnd -= toEnd; } else { toEnd -= frEnd; frEnd -= frEnd; } //fprintf(graph, "hangs2- fr %d-%d to %d-%d\n", // frBgn, frEnd, toBgn, toEnd); // Compute the alignment between the two regions, and convert to a cigar string. frLen -= (frBgn + frEnd); toLen -= (toBgn + toEnd); sprintf(cigar, "%dM", (frLen + toLen) / 2); // Used to be 'm', Bandage complained about it not being 'M'. // The overlap should now have one of: // frBgn == toEnd == 0 -- to has an overlap to fr // frEnd == toBgn == 0 -- fr has an overlap to to // // If not, the overlap is inconsistent with the tigs; it implies the two tigs overlap in their // entirety. // GFA requires that the overlap be between the end of the first read and the start of the second read. // Flip the order if needed. if (frTigID == toTigID) { fprintf(stderr, "L\ttig%08u\t%c\ttig%08d\t%c\t%s circular\n", frTigID, (frTigFwd) ? '+' : '-', toTigID, (toTigFwd) ? '+' : '-', cigar); continue; } if ((toBgn == 0) && (frEnd == 0)) { fprintf(graph, "L\ttig%08u\t%c\ttig%08d\t%c\t%s\n", frTigID, (frTigFwd) ? '+' : '-', toTigID, (toTigFwd) ? '+' : '-', cigar); continue; } if ((frBgn == 0) && (toEnd == 0)) { fprintf(graph, "L\ttig%08u\t%c\ttig%08d\t%c\t%s\n", toTigID, (toTigFwd) ? '+' : '-', frTigID, (frTigFwd) ? '+' : '-', cigar); continue; } // Inconsistent edge. fprintf(stderr, "L\ttig%08u\t%c\ttig%08d\t%c\t%s inconsistent\n", frTigID, (frTigFwd) ? '+' : '-', toTigID, (toTigFwd) ? '+' : '-', cigar); } edges.clear(); // Make valgrind slightly happier. vertices.clear(); delete [] cigar; delete tigStore; gkpStore->gkStore_close(); exit(0); } canu-1.3/src/bogart/buildGraph.mk000066400000000000000000000007371272335422300167740ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := buildGraph SOURCES := buildGraph.C SRC_INCDIRS := .. ../AS_UTL ../stores TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/bogus/000077500000000000000000000000001272335422300142145ustar00rootroot00000000000000canu-1.3/src/bogus/bogus-genome.css000066400000000000000000000376001272335422300173230ustar00rootroot00000000000000html, body { height: 98%; } .nav { vertical-align: middle; z-index: 10; } input.icon { display: -moz-inline-stack; display: inline-block; zoom: 1; *display: inline; } div.container { position: absolute; z-index: 0; } div.dragWindow { position:absolute; overflow: hidden; z-index: 1; cursor: url("openhand.cur"); } div.locationTrap { position: absolute; background-color: #BDD7FF; border-color: white white #BDD7FF white; border-style: solid; width: 0px; height: 0px; line-height: 0px; z-index: -10; } div.locationThumb { position: absolute; top: 0px; /* if you change this border from 2px, change GenomeView.showTrap */ border: 2px solid red; margin: 0px -2px 0px -2px; cursor: url("openhand.cur"); } div.locationThumb.dojoMoveItem { cursor: url("closedhand.cur"); } div.overview { width: 100%; border-style: solid; border-width: 5px 0px 5px 0px; border-color: #aaa; color:#aaa; font-family: sans-serif; text-align: center; z-index: -5; } div.block { position: absolute; overflow: visible; top: 0px; height: 100%; } div.track { position: absolute; left: 0px; width: 100%; z-index: 5; } .track.dojoDndItemOver { cursor: inherit; } .track.dojoDndItemBefore { border-top: 3px solid #999; margin-top: -3px; } .track.dojoDndItemAfter { border-bottom: 3px solid #999; margin-bottom: -3px; } div#static_track { top: 0px; position: absolute; background-color: #f0f0f0; z-index: 20; } div.gridline { position: absolute; top: 0px; height: 100%; border-style: none none none solid; border-width: 1px; border-color: #ddd; } div.pos-label { position: absolute; left: 0px; background-color: #ddd; z-index: 100; padding: 4px; font-family: sans-serif; } div.overview-pos { position: absolute; left: 0px; color: black; padding-left: 4px; font-family: sans-serif; border-style: solid; border-color: black; border-width: 0px 0px 0px 1px; } div.blank-block { font: sans-serif; position: absolute; overflow: visible; top: 0px; height: 100%; background-color: #eee; z-index: 19; } div.sequence { position: absolute; left: 0px; font-family: monospace; letter-spacing: 2px; padding-left: 2px; } /* div.minus-feature:before { content: attr(fName); display: block; } div.plus-feature:before { content: attr(fName); display: inline; } */ div.track-label { font-family: sans-serif; z-index: 20; background-color: #BDD7FF; border: 2px #2b434c solid; color: #2b434c; padding: 5px; cursor: pointer; } div.tracklist-label { font-family: sans-serif; z-index: 20; background-color: #BDD7FF; border: 2px #2b434c solid; color: #2b434c; cursor: pointer; padding: 3px; } /* commented for now, multi-select too confusing? .tracklist-container.dojoDndItemSelected { background: #ddf; } .tracklist-container.dojoDndItemAnchor { background: #ddf; } */ div.tracklist-container { padding: 5px; margin-top: -3px; margin-bottom: -3px; } .tracklist-container.dojoDndItemBefore { border-top: 3px solid #999; padding-top: 2px; } .tracklist-container.dojoDndItemAfter { border-bottom: 3px solid #999; padding-bottom: 2px; } .feature-label { position: absolute; font-family: monospace; border: 0px; margin: -2px 0px 0px 0px; /* padding: 0px 0px 2px 0px; for more space below labels */ padding: 0px 0px 0px 0px; /* font-size: 80%; */ white-space: nowrap; background-color: #eee; z-index: 10; cursor: pointer; } .basic, .plus-basic, .minus-basic { position: absolute; cursor: pointer; z-index: 10; min-width: 1px; } div.basic-hist { position: absolute; z-index: 10; } .plus-feature, .minus-feature { position:absolute; height: 8px; background-repeat: repeat-x; cursor: pointer; min-width: 1px; z-index: 10; } .plus-feature { background-image: url('img/plus-chevron3.png'); } .minus-feature { background-image: url('img/minus-chevron3.png'); } div.feature-hist { position: absolute; background-color: blue; border-color: lightblue; border-style: solid; border-width: 1px; z-index: 10; } .plus-feature2, .minus-feature2 { position:absolute; height: 15px; background-repeat: repeat-x; cursor: pointer; min-width: 1px; z-index: 10; } .plus-feature2 { background-image: url('img/plus-herringbone16.png'); } .minus-feature2 { background-image: url('img/minus-herringbone16.png'); } div.feature2-hist { position: absolute; background-color: #9f9; border-color: #ada; border-style: solid; border-width: 1px; z-index: 10; } .plus-feature3, .minus-feature3 { position:absolute; height: 8px; background-repeat: repeat-x; cursor: pointer; min-width: 1px; z-index: 10; } .plus-feature3 { background-image: url('img/plus-chevron.png'); } .minus-feature3 { background-image: url('img/minus-chevron.png'); } div.feature3-hist { position: absolute; background-color: yellow; border-color: black; border-style: solid; border-width: 1px; z-index: 10; } .plus-feature4, .minus-feature4 { position:absolute; height: 12px; background-repeat: repeat-x; cursor: pointer; min-width: 1px; z-index: 10; } .plus-feature4 { background-image: url('img/plus-pacman.png'); } .minus-feature4 { background-image: url('img/minus-pacman.png'); } div.feature4-hist { position: absolute; background-color: yellow; border-color: black; border-style: solid; border-width: 1px; z-index: 10; } .plus-feature5, .minus-feature5 { position:absolute; height: 8px; background-repeat: repeat-x; cursor: pointer; min-width: 1px; z-index: 10; } .plus-feature5 { background-image: url('img/plus-chevron2.png'); } .minus-feature5 { background-image: url('img/minus-chevron2.png'); } div.feature5-hist { position: absolute; background-color: blue; border-color: lightblue; border-style: solid; border-width: 1px; z-index: 10; } div.exon-hist { position: absolute; background-color: #4B76E8; border-style: solid; border-color: #00f; border-width: 1px; z-index: 10; } .plus-exon, .minus-exon { position: absolute; height: 5px; background-color: #4B76E8; border-style: solid; border-color: #00f; border-width: 1px; cursor: pointer; z-index: 10; } div.est-hist { position: absolute; background-color: #ED9185; border-style: solid; border-color: #c33; border-width: 1px; z-index: 10; } .plus-est, .minus-est { position: absolute; height: 5px; background-color: #ED9185; border-style: solid; border-color: #c33; border-width: 1px; cursor: pointer; z-index: 10; } .plus-dblhelix, .minus-dblhelix { position:absolute; height: 11px; background-image: url('img/dblhelix-red.png'); background-repeat: repeat-x; min-width: 1px; cursor: pointer; z-index: 10; } div.dblhelix-hist { position: absolute; background-color: #fcc; border-color: #daa; border-style: solid; border-width: 1px; z-index: 10; } .plus-helix, .minus-helix { position:absolute; height: 12px; background-image: url('img/helix3-green.png'); background-repeat: repeat-x; min-width: 1px; cursor: pointer; z-index: 10; } div.helix-hist { position: absolute; background-color: #cfc; border-color: #ada; border-style: solid; border-width: 1px; z-index: 10; } .loops { position:absolute; height: 13px; background-image: url('img/loops.png'); background-repeat: repeat-x; cursor: pointer; } .plus-cds0, .plus-cds1, .plus-cds2, .minus-cds0, .minus-cds1, .minus-cds2 { position:absolute; height: 13px; background-repeat: repeat-x; cursor: pointer; min-width: 1px; } .plus-cds0 { background-image: url('img/plus-cds0.png'); } .plus-cds1 { background-image: url('img/plus-cds1.png'); } .plus-cds2 { background-image: url('img/plus-cds2.png'); } .minus-cds0 { background-image: url('img/minus-cds0.png'); } .minus-cds1 { background-image: url('img/minus-cds1.png'); } .minus-cds2 { background-image: url('img/minus-cds2.png'); } div.cds-hist { position: absolute; background-color: #fcc; border-color: #daa; border-style: solid; border-width: 1px; z-index: 10; } .topbracket { position:absolute; height: 8px; border-style: solid solid none solid; /* border-width: 2px 2px 0px 2px; */ border-width: 2px; border-color: orange; /* margin-top: 2px */ cursor: pointer; } .bottombracket { position:absolute; height: 8px; border-style: none solid solid solid; border-width: 2px; border-color: green; cursor: pointer; } .hourglass { position:absolute; height: 0px; border-style: solid; border-width: 6px 3px 6px 3px; cursor: pointer; } .plus-triangle { position:absolute; height: 0px; border-style: solid; border-width: 6px 3px 0px 3px; cursor: pointer; } .minus-triangle { position:absolute; height: 0px; border-style: solid; border-width: 0px 3px 6px 3px; cursor: pointer; } .triangle { position:absolute; height: 0px; border-style: solid; border-width: 6px 0px 0px 0px; cursor: pointer; } .hgred { border-color: #f99 white #f99 white; } div.hgred-hist { position: absolute; background-color: #daa; border-color: #d44; border-style: solid; border-width: 1px; z-index: 10; } .hgblue { border-color: #99f white #99f white; } div.hgblue-hist { position: absolute; background-color: #aad; border-color: #99f; border-style: solid; border-width: 1px; z-index: 10; } .ibeam { position:absolute; height: 2px; background-color: blue; border-style: solid; border-width: 8px 4px 8px 4px; border-color: white blue white blue; cursor: pointer; } div.transcript-hist { position: absolute; background-color: #ddd; border-color: #FF9185; border-style: solid; border-width: 1px; z-index: 10; } .transcript, .plus-transcript, .minus-transcript { position: absolute; height: 4px; margin-top: 4px; margin-bottom: 4px; background-color: #999; z-index: 6; min-width: 1px; cursor: pointer; } .plus-transcript-arrowhead { position: absolute; /* border stuff seems slow height: 0px; width: 0px; margin-top: -4px; border-style: solid; border-color: white white white #999; border-width: 6px 0px 6px 10px; */ margin-top: -4px; width: 12px; height: 12px; background-image: url('img/plus-transcript-head.png'); background-repeat: no-repeat; } .minus-transcript-arrowhead { position: absolute; /* border stuff seems slow height: 0px; width: 0px; margin-top: -4px; border-style: solid; border-color: white #999 white white; border-width: 6px 10px 6px 0px; */ margin-top: -4px; width: 12px; height: 12px; background-image: url('img/minus-transcript-head.png'); background-repeat: no-repeat; } .plus-transcript-CDS, .minus-transcript-CDS { position: absolute; height: 12px; margin-top: -4px; background-image: url('img/cds.png'); background-repeat: repeat-x; /* border-width: 2px 0px 3px 0px; border-style: solid; border-color: white; background-color: #FF9185; border-style: solid; border-color: #00f; border-width: 1px;*/ cursor: pointer; z-index: 10; min-width: 1px; } .plus-transcript-exon, .minus-transcript-exon, .plus-transcript-UTR, .minus-transcript-UTR, .plus-transcript-five_prime_UTR, .minus-transcript-five_prime_UTR, .plus-transcript-three_prime_UTR, .minus-transcript-three_prime_UTR { position: absolute; height: 4px; margin-top: -2px; background-color: #B66; border-style: solid; border-color: #D88; border-width: 2px 0px 2px 0px; z-index: 8; min-width: 1px; cursor: pointer; } .generic_parent, .plus-generic_parent, .minus-generic_parent { position: absolute; height: 4px; margin-top: 2px; margin-bottom: 2px; background-color: #AAA; z-index: 6; min-width: 1px; cursor: pointer; } div.generic_parent-hist { position: absolute; background-color: #ddd; border-color: #555; border-style: solid; border-width: 1px; z-index: 10; } .match_part, .plus-match_part, .minus-match_part { position: absolute; height: 4px; margin-top: -2px; background-color: #66B; border-style: solid; border-color: #88D; border-width: 2px 0px 2px 0px; z-index: 8; min-width: 1px; cursor: pointer; } .generic_part_a, .plus-generic_part_a, .minus-generic_part_a { position: absolute; height: 4px; margin-top: -2px; background-color: #6B6; border-style: solid; border-color: #8D8; border-width: 2px 0px 2px 0px; z-index: 8; min-width: 1px; cursor: pointer; } /* * Definitions for jbrowse to show BOGUS and BOGUSNESS results. */ */ * BOGUS classes */ .frag_align { position: absolute; height: 4px; margin-top: 2px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; background-color: #AAA; } .plus-frag_align { position: absolute; height: 4px; margin-top: 2px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; background-color: #EB2A2A; } .minus-frag_align { position: absolute; height: 4px; margin-top: 2px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; background-color: #2AEB2A; } div.frag_align-hist { position: absolute; background-color: #ddd; border-color: #555; border-style: solid; border-width: 1px; z-index: 10; } .rept_interval { position: absolute; height: 4px; margin-top: 2px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; background-color: #1010bb; } .uniq_interval { position: absolute; height: 4px; margin-top: 2px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; background-color: #10bb10; } .weak_interval { position: absolute; height: 8px; margin-top: -3px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; border-color: #bb1010; border-style: solid; border-width: 1px; } .sepr_interval { position: absolute; height: 8px; margin-top: -3px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; border-color: #bb1010; border-style: solid; border-width: 1px; } /* * BOGUSNESS classes */ .bogusness_span { position: absolute; height: 4px; margin-top: 2px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; background-color: #AAA; } .plus-bogusness_match { position: absolute; height: 8px; margin-top: -2px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; background-color: #ee2222; } .minus-bogusness_match { position: absolute; height: 8px; margin-top: -2px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; background-color: #22ee22; } .plus-bogusness_repeat { position: absolute; height: 8px; margin-top: -2px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; border-color: 10bb10; border-style: solid; border-width: 2px; background-color: #ee2222; } .minus-bogusness_repeat { position: absolute; height: 8px; margin-top: -2px; margin-bottom: 2px; z-index: 6; min-width: 1px; cursor: pointer; border-color: #bb1010; border-style: solid; border-width: 2px; background-color: #22ee22; } canu-1.3/src/bogus/bogus-genome.json000066400000000000000000000051471272335422300175050ustar00rootroot00000000000000 { "description": "Bogus Results", "db_adaptor": "Bio::DB::SeqFeature::Store", "db_args": { "-adaptor": "memory", "-dir": "."}, "TRACK DEFAULTS": { "class": "feature" }, "tracks": [ { "track": "BOGUS_raw_input", "key": "BOGUS raw input", "feature": ["bogus_raw_input"], "autocomplete": "all", "class": "frag_align" }, { "track": "BOGUS_span_input", "key": "BOGUS span input", "feature": ["bogus_span_input"], "autocomplete": "all", "class": "frag_align" }, { "track": "BOGUS_rept_input", "key": "BOGUS rept input", "feature": ["bogus_rept_input"], "autocomplete": "all", "class": "frag_align" }, { "track": "BOGUS_uniq_input", "key": "BOGUS uniq input", "feature": ["bogus_uniq_input"], "autocomplete": "all", "class": "frag_align" }, { "track": "BOGUS_rept_interval", "key": "BOGUS rept interval", "feature": ["bogus_rept_interval"], "autocomplete": "all", "class": "rept_interval", "subfeatures": true, "subfeature_classes": {"bogus_sepr_interval": "sepr_interval"}, "clientConfig": { "labelScale": 1000000 } }, { "track": "BOGUS_uniq_interval", "key": "BOGUS uniq interval", "feature": ["bogus_uniq_interval"], "autocomplete": "all", "class": "uniq_interval", "subfeatures": true, "subfeature_classes": {"bogus_weak_interval": "weak_interval"}, "clientConfig": { "labelScale": 1000000 } }, { "track": "BOGUSNESS", "key": "BOGUSNESS", "feature": ["bogusness_span"], "autocomplete": "all", "class": "bogusness_span", "subfeatures": true, "subfeature_classes": {"bogusness_match": "bogusness_match"}, "clientConfig": { "labelScale": 1000000 } }, { "track": "BOGUSNESS_repeat", "key": "BOGUSNESS repeat", "feature": ["bogusness_repeat"], "autocomplete": "all", "class": "bogusness_repeat" } ] } canu-1.3/src/bogus/bogus-run.sh000066400000000000000000000033611272335422300164740ustar00rootroot00000000000000#!/bin/sh # Build ideal unitigs given a fasta or frg file. # Align ideal unitigs back to the reference with nucmer. # Generate a mummerplot. # # ASSUMES all programs are in your path. # mummer: nucmer, mummerplot # kmer: snapper2, convertToExtent # wgs-assembler: bogus # Edit reference to make defline have only one work. This is necessary because snapper reports the # whole line, but bogus truncates to the first word. FAS="../FRAGS/porphyromonas_gingivalis_w83.flx.3200bp.0900bp.FJRUAFO0.fasta" REF="../AE015924.fasta" if [ $# -gt 0 ] ; then FAS=$1 shift fi if [ $# -gt 0 ] ; then REF=$1 shift fi OUT=`echo $FAS | tr '/' ' ' | awk '{ print $NF}' | sed s/.fasta//` if [ ! -e $FAS ] ; then echo "Failed to find FASTA $FAS" exit fi if [ ! -e $REF ] ; then echo "Failed to find REFERENCE $REF" exit fi if [ `cat $REF | wc -l` != 2 ] ; then echo "REFERENCE is multi-line FASTA, sequence must be on one line." exit fi if [ ! -e $OUT.snapper ] ; then snapper2 \ -queries $FAS \ -genomic $REF \ -minmatchidentity 94 -minmatchcoverage 1 -verbose -mersize 16 \ -output $OUT.snapper fi if [ ! -e $OUT.snapper.extent ] ; then convertToExtent \ < $OUT.snapper \ | grep -v ^cDNAid \ | sort -k7n \ > $OUT.snapper.extent fi if [ ! -e $OUT.ideal.fasta ] ; then bogus \ -snapper $OUT.snapper.extent \ -reference $REF \ -output $OUT.ideal fi if [ ! -e $OUT.ideal.fasta ] ; then echo "No fasta output from bogus, not mapping to reference." exit fi if [ ! -e $OUT.ideal.delta ] ; then nucmer --maxmatch --coords -p $OUT.ideal \ $REF \ $OUT.ideal.fasta fi if [ ! -e $OUT.ideal.png ] ; then mummerplot --layout --filter -p $OUT.ideal -t png $OUT.ideal.delta fi canu-1.3/src/bogus/bogus.C000066400000000000000000000623721272335422300154510ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/bogus.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-09 to 2014-DEC-23 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "bogusUtil.H" class longestAlignment { public: longestAlignment() { bgn = end = len = num = 0; rptBgn = rptEnd = frgLen = 0; }; int32 bgn; // Begin coord on the fragment for this region int32 end; // End coord on the fragment for this region int32 len; // Length of the match int32 num; // Number of matches on this region int32 rptBgn; int32 rptEnd; int32 frgLen; }; // MUCH, much easier to modularize the code if all this stuff is globally available. vector refList; map refMap; vector longest; vector genome; map IIDmap; // Maps an ID string to an IID. vector IIDname; // Maps an IID to an ID string. vector IIDcount; // Maps an IID to the number of alignments intervalList REPT; intervalList UNIQ; bool *REPTvalid = NULL; bool *UNIQvalid = NULL; int32 *REPTvalidParent = NULL; int32 *UNIQvalidParent = NULL; FILE *gffOutput = NULL; FILE *intervalOutput = NULL; static void writeInputsAsGFF3(char *outputPrefix) { for (uint32 i=0; ifrgEnd - A->frgBgn) { longest[frgIID].bgn = A->frgBgn; longest[frgIID].end = A->frgEnd; longest[frgIID].len = A->frgEnd - A->frgBgn; longest[frgIID].num = 0; indelRate = (A->genEnd - A->genBgn) / (double)(A->frgEnd - A->frgBgn); } if (longest[frgIID].frgLen < A->frgEnd) { longest[frgIID].rptBgn = 0; longest[frgIID].rptEnd = A->frgEnd; longest[frgIID].frgLen = A->frgEnd; } } // Now that we know the longest, count the number of matches that have approximately the same // span. If that count is one, mark all but that span as repeat, otherwise, mark everthing as // repeat. assert(longest[frgIID].num == 0); for (uint32 i=bgn; iisRepeat = true; if (((longest[frgIID].bgn - alignWobble <= A->frgBgn) && (A->frgBgn <= longest[frgIID].bgn + alignWobble)) && ((longest[frgIID].end - alignWobble <= A->frgEnd) && (A->frgEnd <= longest[frgIID].end + alignWobble))) { longest[frgIID].num++; A->isRepeat = false; } } assert(longest[frgIID].num > 0); // More than one longest? Then all alignments are repeats. if (longest[frgIID].num > 1) for (uint32 i=bgn; iisRepeat == false) { assert(LONG == 0L); LONG = A; continue; } if ((A->frgBgn < alignWobble) && (A->frgEnd > longest[frgIID].rptBgn)) // First N bases of the read is part of a repeat longest[frgIID].rptBgn = A->frgEnd; if ((A->frgEnd > longest[frgIID].frgLen - alignWobble) && (A->frgBgn < longest[frgIID].rptEnd)) // Last N bases of the read is part of a repeat longest[frgIID].rptEnd = A->frgBgn; } assert(LONG != 0L); // Indel screws up this compute. We have the repeat region marked on the read, and need to transfer it to the genome. // We estimate, globally, the indel rate, and scale the repeat region length. if (longest[frgIID].rptBgn > 0) { uint32 rptLen = (longest[frgIID].rptBgn - 0) * indelRate; if (rptLen > 0) { #ifdef DEBUG fprintf(stderr, "addAlignment()-- RPTBGN longest %u-%u frg %u at frg %u-%u chn %u-%u gen %u %u-%u rptlen %u indelrate %f\n", longest[frgIID].rptBgn, longest[frgIID].rptEnd, LONG->frgIID, LONG->frgBgn, LONG->frgEnd, LONG->chnBgn, LONG->chnEnd, LONG->genIID, LONG->genBgn, LONG->genEnd, rptLen, indelRate); #endif addAlignment(genome, LONG->frgIID, 0, longest[frgIID].rptBgn, false, LONG->chnBgn, LONG->chnBgn + rptLen, LONG->identity, LONG->genIID, LONG->genBgn, LONG->genBgn + rptLen); #ifdef DEBUG uint32 ii = genome.size() - 1; fprintf(stderr, "addAlignment()-- RPTBGN FINI frg %u at frg %u-%u chn %u-%u gen %u %u-%u\n", genome[ii].frgIID, genome[ii].frgBgn, genome[ii].frgEnd, genome[ii].chnBgn, genome[ii].chnEnd, genome[ii].genIID, genome[ii].genBgn, genome[ii].genEnd); #endif } } if (longest[frgIID].rptEnd < LONG->frgEnd) { uint32 rptLen = (LONG->frgEnd - longest[frgIID].rptEnd) * indelRate; if (rptLen > 0) { #ifdef DEBUG fprintf(stderr, "addAlignment()-- RPTEND longest %u-%u frg %u at frg %u-%u chn %u-%u gen %u %u-%u rptlen %u indelrate %f\n", longest[frgIID].rptBgn, longest[frgIID].rptEnd, LONG->frgIID, LONG->frgBgn, LONG->frgEnd, LONG->chnBgn, LONG->chnEnd, LONG->genIID, LONG->genBgn, LONG->genEnd, rptLen, indelRate); #endif addAlignment(genome, LONG->frgIID, longest[frgIID].rptEnd, LONG->frgEnd, false, LONG->chnEnd - rptLen, LONG->chnEnd, LONG->identity, LONG->genIID, LONG->genEnd - rptLen, LONG->genEnd); #ifdef DEBUG uint32 ii = genome.size() - 1; fprintf(stderr, "addAlignment()-- RPTEND FINI frg %u at frg %u-%u chn %u-%u gen %u %u-%u\n", genome[ii].frgIID, genome[ii].frgBgn, genome[ii].frgEnd, genome[ii].chnBgn, genome[ii].chnEnd, genome[ii].genIID, genome[ii].genBgn, genome[ii].genEnd); #endif } } } bgn = end; } // Over all matches, by fragment #ifdef DEBUG for (uint32 ii=0; ii 0) && (minFrags <= refcnt) && (minLength <= refend - refbgn)) { fprintf(intervalOutput, "%s\t%8"F_S64P"\t%8"F_S64P"\tREPT\t"F_S64"%s\n", refhdr, refbgn, refend, refcnt, (REPTvalid[ir]) ? "" : " weak"); if (REPTvalid[ir]) fprintf(gffOutput, "%s\t.\tbogus_rept_interval\t"F_S64"\t"F_S64"\t.\t.\t.\tID=REPT%04d;fragCount="F_S64"\n", refhdr, refbgn, refend, ir, refcnt); else fprintf(gffOutput, "%s\t.\tbogus_weak_interval\t"F_S64"\t"F_S64"\t.\t.\t.\tParent=UNIQ%04d;fragCount="F_S64"\n", refhdr, refbgn, refend, REPTvalidParent[ir], refcnt); } ir++; } else { for (uint32 rr=0; rr 0) && (minFrags <= refcnt) && (minLength <= refend - refbgn)) { fprintf(intervalOutput, "%s\t%8"F_S64P"\t%8"F_S64P"\tUNIQ\t"F_S64"%s\n", refhdr, refbgn, refend, refcnt, (UNIQvalid[iu]) ? "" : " separation"); if (UNIQvalid[iu]) fprintf(gffOutput, "%s\t.\tbogus_uniq_interval\t"F_S64"\t"F_S64"\t.\t.\t.\tID=UNIQ%04d;fragCount="F_S64"\n", refhdr, refbgn, refend, iu, refcnt); else fprintf(gffOutput, "%s\t.\tbogus_sepr_interval\t"F_S64"\t"F_S64"\t.\t.\t.\tParent=REPT%04d;fragCount="F_S64"\n", refhdr, refbgn, refend, UNIQvalidParent[iu], refcnt); } iu++; } } fclose(gffOutput); fclose(intervalOutput); // See CVS version 1.3 for writing rept/uniq fasta return(0); } canu-1.3/src/bogus/bogus.json.README000066400000000000000000000010251272335422300171600ustar00rootroot00000000000000JSON config for viewing BOGUS and BOGUSNESS in jbrowse. Unzip jbrowse-1.1.zip to the directory with bogus/bogusness results. Copy bogus.json to jbrowse/ Append bogus.css to jbrowse/genome.css In jbrowse/: bin/prepare-refseqs.pl -fasta ../reference.fasta bin/biodb-to-json.pl --conf bogus.json NOTE that the reference sequences CANNOT have a space in the defline. 'bogus' will silently remove these, but jbrowse will not. ----- # SCALING on ZOOM - in json "clientConfig": { "labelScale": 1000000 } canu-1.3/src/bogus/bogus.mk000066400000000000000000000007571272335422300156750ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := bogus SOURCES := bogus.C \ bogusUtil.C SRC_INCDIRS := .. ../AS_UTL ../stores TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/bogus/bogusUtil.C000066400000000000000000000214711272335422300163020ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_bogusUtil.C * src/bogart/AS_BAT_bogusUtil.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2014-JAN-23 * are Copyright 2010-2011,2013-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2011-JUN-27 * are Copyright 2011 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2014-DEC-23 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "bogusUtil.H" #define MAX_GENOME_SIZE_INPUT 256 * 1024 * 1024 bool byFragmentID(const genomeAlignment &A, const genomeAlignment &B) { return(A.frgIID < B.frgIID); } bool byGenomePosition(const genomeAlignment &A, const genomeAlignment &B) { if (A.chnBgn < B.chnBgn) // A clearly before B. return(true); if (A.chnBgn > B.chnBgn) // A clearly after B. return(false); if ((A.isRepeat == false) && (B.isRepeat == true)) // Start at the same spot, put unique stuff first return(true); // Note the second condition above. If both A and B are repeat=false, byGenomePosition(A,B) and // byGenomePosition(B,A) bth return true, logically impossible. return(false); } void addAlignment(vector &genome, int32 frgIID, int32 frgBgn, int32 frgEnd, bool isReverse, int32 chnBgn, int32 chnEnd, double identity, int32 genIID, int32 genBgn, int32 genEnd) { genomeAlignment A; A.frgIID = frgIID; A.frgBgn = frgBgn; A.frgEnd = frgEnd; A.genIID = genIID; A.genBgn = genBgn; A.genEnd = genEnd; A.chnBgn = chnBgn; A.chnEnd = chnEnd; A.identity = identity; A.isReverse = isReverse; A.isSpanned = false; A.isRepeat = true; assert(A.frgBgn < A.frgEnd); assert(A.genBgn < A.genEnd); genome.push_back(A); } void loadNucmer(char *nucmerName, vector &genome, map &IIDmap, vector &IIDname, vector &refList, map &refMap, double minIdentity) { FILE *inFile = 0L; char inLine[1024]; errno = 0; inFile = fopen(nucmerName, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", nucmerName, strerror(errno)), exit(1); fprintf(stderr, "Loading alignments from '%s'\n", nucmerName); // First FOURE lines are header fgets(inLine, 1024, inFile); // (file paths) fgets(inLine, 1024, inFile); // NUCMER fgets(inLine, 1024, inFile); // (blank) fgets(inLine, 1024, inFile); // (header) // Scan the header line, counting the number of columns uint32 nCols = 0; uint32 wIdent = 0; for (uint32 xx=0; inLine[xx]; xx++) { if ((wIdent == 0) && (inLine[xx+0] == '[') && (inLine[xx+1] == '%') && (inLine[xx+2] == ' ')) wIdent = nCols; if (inLine[xx] == '[') nCols++; } // Read the first line. fgets(inLine, 1024, inFile); chomp(inLine); while (!feof(inFile)) { for (uint32 xx=0; inLine[xx]; xx++) if (inLine[xx] == '|') inLine[xx] = ' '; splitToWords W(inLine); genomeAlignment A; string gID = W[nCols - 1]; // TAGS is the last header column, string fID = W[nCols - 0]; // but read ID is in column +1 from there. if (IIDmap.find(fID) == IIDmap.end()) { IIDname.push_back(fID); IIDmap[fID] = IIDname.size() - 1; } // Unlike snapper, these are already in base-based coords. A.frgIID = IIDmap[fID]; A.frgBgn = W(2); A.frgEnd = W(3); A.genIID = refMap[gID]; A.genBgn = W(0); A.genEnd = W(1); A.chnBgn = refList[A.genIID].rschnBgn + A.genBgn; A.chnEnd = refList[A.genIID].rschnBgn + A.genEnd; A.identity = atof(W[wIdent]); A.isReverse = false; A.isSpanned = false; A.isRepeat = true; if (A.frgBgn > A.frgEnd) { A.frgBgn = W(3); A.frgEnd = W(2); A.isReverse = true; } if ((A.frgBgn >= A.frgEnd) || (A.genBgn >= A.genEnd)) { fprintf(stderr, "ERROR: %s\n", inLine); if (A.frgBgn >= A.frgEnd) fprintf(stderr, "ERROR: frgBgn,frgEnd = %u,%u\n", A.frgBgn, A.frgEnd); if (A.genBgn >= A.genEnd) fprintf(stderr, "ERROR: genBgn,genEnd = %u,%u\n", A.genBgn, A.genEnd); } assert(A.frgBgn < A.frgEnd); assert(A.genBgn < A.genEnd); if (A.identity < minIdentity) goto nextNucmerLine; genome.push_back(A); nextNucmerLine: fgets(inLine, 1024, inFile); chomp(inLine); } fclose(inFile); } void loadSnapper(char *snapperName, vector &genome, map &IIDmap, vector &IIDname, vector &refList, map &refMap, double minIdentity) { FILE *inFile = 0L; char inLine[1024]; errno = 0; inFile = fopen(snapperName, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", snapperName, strerror(errno)), exit(1); // Read the first line fgets(inLine, 1024, inFile); chomp(inLine); while (!feof(inFile)) { splitToWords W(inLine); genomeAlignment A; string fID = W[0]; string gID = W[5]; if ((W[1][0] < '0') || (W[1][0] > '9')) // Skip header lines. goto nextSnapperLine; if (IIDmap.find(fID) == IIDmap.end()) { IIDname.push_back(fID); IIDmap[fID] = IIDname.size() - 1; } // "+1" -- Convert from space-based coords to base-based coords. A.frgIID = IIDmap[fID]; A.frgBgn = W(3) + 1; A.frgEnd = W(4); A.genIID = refMap[gID]; A.genBgn = W(6) + 1; A.genEnd = W(7); A.chnBgn = refList[A.genIID].rschnBgn + A.genBgn; A.chnEnd = refList[A.genIID].rschnBgn + A.genEnd; A.identity = atof(W[8]); A.isReverse = false; A.isSpanned = false; A.isRepeat = true; if (A.frgBgn > A.frgEnd) { A.frgBgn = W(4) + 1; A.frgEnd = W(3); A.isReverse = true; } assert(A.frgBgn < A.frgEnd); assert(A.genBgn < A.genEnd); if (A.identity < minIdentity) goto nextSnapperLine; genome.push_back(A); nextSnapperLine: fgets(inLine, 1024, inFile); chomp(inLine); } fclose(inFile); } void loadReferenceSequence(char *refName, vector &refList, map &refMap) { int32 reflen = 0; int32 refiid = 0; errno = 0; FILE *F = fopen(refName, "r"); if (errno) fprintf(stderr, "Failed to open reference sequences in '%s': %s\n", refName, strerror(errno)), exit(1); char *refhdr = new char [1024]; char *refseq = new char [MAX_GENOME_SIZE_INPUT]; fgets(refhdr, 1024, F); chomp(refhdr); fgets(refseq, MAX_GENOME_SIZE_INPUT, F); chomp(refseq); while (!feof(F)) { if (refhdr[0] != '>') { fprintf(stderr, "ERROR: reference sequences must be one per line.\n"); exit(1); } for (uint32 i=0; refhdr[i]; i++) { refhdr[i] = refhdr[i+1]; // remove '>' if (isspace(refhdr[i])) // stop at first space refhdr[i] = 0; } int32 rl = strlen(refseq); refMap[refhdr] = refiid; refList.push_back(referenceSequence(reflen, reflen + rl, rl, refhdr)); reflen += rl + 1024; refiid++; fgets(refhdr, 1024, F); chomp(refhdr); fgets(refseq, MAX_GENOME_SIZE_INPUT, F); chomp(refseq); } fclose(F); } canu-1.3/src/bogus/bogusUtil.H000066400000000000000000000075641272335422300163160ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/AS_BAT_bogusUtil.H * src/bogart/AS_BAT_bogusUtil.H * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-09 to 2014-DEC-23 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef INCLUDE_BOGUSUTIL #define INCLUDE_BOGUSUTIL #include "AS_global.H" #include "splitToWords.H" #include "intervalList.H" #include #include #include #include using namespace std; class genomeAlignment { public: genomeAlignment() { frgIID = frgBgn = frgEnd = 0; genIID = 0; genBgn = genEnd = 0; identity = 0.0; isDeleted = isReverse = isSpanned = isRepeat = false; }; int32 frgIID; int32 frgBgn; int32 frgEnd; int32 genIID; // Position in the actual sequence int32 genBgn; int32 genEnd; int32 chnBgn; // Position in the chained sequences int32 chnEnd; double identity; // Percent identity of the alignment bool isDeleted; // Used by bogusness bool isReverse; bool isSpanned; bool isRepeat; }; class referenceSequence { public: referenceSequence(int32 cb, int32 ce, int32 rl, char *cn) { rschnBgn = cb; rschnEnd = ce; rsrefLen = rl; assert(strlen(cn) < 256); strcpy(rsrefName, cn); }; int32 rschnBgn; int32 rschnEnd; int32 rsrefLen; char rsrefName[256]; }; bool byFragmentID(const genomeAlignment &A, const genomeAlignment &B); bool byGenomePosition(const genomeAlignment &A, const genomeAlignment &B); void addAlignment(vector &genome, int32 frgIID, int32 frgBgn, int32 frgEnd, bool isReverse, int32 chnBgn, int32 chnEnd, double identity, int32 genIID, int32 genBgn, int32 genEnd); void loadNucmer(char *nucmerName, vector &genome, map &IIDmap, vector &IIDname, vector &refList, map &refMap, double minIdentity); void loadSnapper(char *snapperName, vector &genome, map &IIDmap, vector &IIDname, vector &refList, map &refMap, double minIdentity); void loadReferenceSequence(char *refName, vector &refList, map &refMap); #endif // INCLUDE_BOGUSUTIL canu-1.3/src/bogus/bogusness-run.pl000066400000000000000000000145621272335422300173730ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/AS_BAT/bogusness-run.pl # # Modifications by: # # Brian P. Walenz from 2010-DEC-02 to 2013-AUG-01 # are Copyright 2010-2011,2013 J. Craig Venter Institute, and # are subject to the GNU General Public License version 2 # # Brian P. Walenz on 2014-DEC-19 # are Copyright 2014 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; use FindBin; my $prefix = undef; # output file name prefix my $SEQ = undef; # input sequences, fasta my $REF = undef; # input reference, fasta my $IDEAL = undef; # input ideal, output from bogus, *.ideal.intervals my $IGFF3 = undef; # input ideal, output from bogus, *.ideal.gff3 # Assumes this script is NOT installed; it looks to this directory to get # jbrowse source and configs. my $src = $FindBin::Bin; # 1.2.1: Needs: (same) # 1.2: Needs: Heap::Simple Heap::Simple::Perl Heap::Simple::XS PerlIO::gzip Devel::Size # 1.1: Needs: BioPerl, JSON, JSON::XS # my $jbrowseVersion = "1.2.1"; my $jbrowse = "$src/jbrowse-$jbrowseVersion.zip\n"; while (scalar(@ARGV) > 0) { my $arg = shift @ARGV; if ($arg eq "-prefix") { $prefix = shift @ARGV; } elsif ($arg eq "-sequence") { $SEQ = shift @ARGV; } elsif ($arg eq "-reference") { $REF = shift @ARGV; } elsif ($arg eq "-ideal") { $IDEAL = shift @ARGV; } else { die "Unknown option '$arg'\n"; } } if (!defined($prefix) || !defined($SEQ) || !defined($REF) || !defined($IDEAL)) { print STDERR "usage: $0 -prefix P -sequence A.fasta -reference R.fasta -ideal I.ideal.intervals\n"; exit(1); } if ($IDEAL =~ m/(.*).intervals/) { $IDEAL = "$1.intervals"; $IGFF3 = "$1.gff3" } die "IDEAL '$IDEAL' not found.\n" if (! -e "$IDEAL"); die "IGFF3 '$IGFF3' not found.\n" if (! -e "$IGFF3"); # # Map assembly to reference # if (! -e "$prefix.delta") { print STDERR "Running nucmer\n"; my $cmd; $cmd .= "nucmer"; $cmd .= " --maxmatch --coords -p $prefix"; $cmd .= " $REF"; $cmd .= " $SEQ"; system($cmd); } if (! -e "$prefix.png") { print STDERR "Running mummerplot\n"; my $cmd; $cmd .= "mummerplot"; $cmd .= " --layout --filter -p $prefix -t png"; #$cmd .= " --layout -p $prefix -t png"; $cmd .= " $prefix.delta"; system($cmd); } # # Run bogusness on that mapping # if (! -e "$prefix.bogusness.out") { print STDERR "Running bogusness\n"; my $cmd; $cmd = "bogusness \\\n"; $cmd .= " -reference $REF \\\n"; $cmd .= " -ideal $IDEAL \\\n"; $cmd .= " -nucmer $prefix.coords \\\n"; $cmd .= " -output $prefix \\\n"; $cmd .= " > $prefix.bogusness.out"; system($cmd); } # # Build wiki-friendly output # if (! -e "$prefix.bogusness.wiki") { my $lastUtg; open(F, "sort -k10n -k18n < $prefix.bogusness.out |") or die "Failed to read sort output.\n"; open(O, "> $prefix.bogusness.wiki") or die "Failed to open '$prefix.bogusness.wiki' for writing.\n"; print O "{| class=\"wikitable\" border=1\n"; print O "! unitig !! align num !! utg coords !! gen coords !! status !! ideal type !! ideal index !! ideal coords !! length !! utg cov !! ideal cov !! annotation\n"; print O "|-\n"; while () { chomp; if (m/^\|\s(utg\d+)\s/) { if (!defined($lastUtg)) { $lastUtg = $1; } if ($lastUtg ne $1) { print O "| colspan=12 bgcolor=#666666 |\n"; print O "|-\n"; $lastUtg = $1; } s!BEGINSin \|\| UNIQ!bgcolor="FireBrick" \| BEGINSin \|\| bgcolor="FireBrick" \| UNIQ!; s!ENDSin \|\| UNIQ!bgcolor="FireBrick" \| ENDSin \|\| bgcolor="FireBrick" \| UNIQ!; s!BEGINSin \|\| REPT!bgcolor="ForestGreen" \| BEGINSin \|\| bgcolor="ForestGreen" \| REPT!; s!ENDSin \|\| REPT!bgcolor="ForestGreen" \| ENDSin \|\| bgcolor="ForestGreen" \| REPT!; s!CONTAINED \|\| UNIQ!bgcolor="Indigo" \| CONTAINED \|\| bgcolor="Indigo" \| UNIQ!; s!CONTAINED \|\| REPT!bgcolor="SteelBlue" \| CONTAINED \|\| bgcolor="SteelBlue" \| REPT!; print O "$_\n"; print O "|-\n"; } } print O "|}\n"; close(O); close(F); } # # Attempt to create a jbrowse instance. # # cd x010 && tar -xf ../jbrowse.tar && mv jbrowse jbrowse-ctg && cd jbrowse-ctg/ && \ # ln -s ../bogusness-ctg.gff3 ../bogus.gff3 . && \ # bin/prepare-refseqs.pl -fasta ../../NC_000913.fasta && \ # bin/biodb-to-json.pl --conf bogus.json && \ # cd ../.. # system("mkdir $prefix.jbrowse"); open(F, "> $prefix.jbrowse/create.sh") or die "Failed to open '$prefix.jbrowse/create.sh' for writing.\n"; print F "#!/bin/sh\n"; print F "\n"; print F "cd $prefix.jbrowse\n"; print F "\n"; print F "unzip -q $jbrowse\n"; print F "\n"; print F "mv jbrowse-$jbrowseVersion/* jbrowse/* .\n"; print F "rm -rf jbrowse-$jbrowseVersion\n"; print F "\n"; print F "cp -p $src/bogus-genome.css genome.css\n"; print F "cp -p $src/bogus-genome.json bogus.json\n"; print F "\n"; print F "ln -s ../$prefix.gff3\n"; print F "\n"; print F "ln -s ../$IGFF3 .\n"; print F "\n"; print F "echo bin/prepare-refseqs.pl -fasta ../$REF\n"; print F "bin/prepare-refseqs.pl -fasta ../$REF\n"; print F "\n"; print F "echo bin/biodb-to-json.pl --conf bogus.json\n"; print F "bin/biodb-to-json.pl --conf bogus.json\n"; print F "\n"; close(F); system("sh $prefix.jbrowse/create.sh"); canu-1.3/src/bogus/bogusness.C000066400000000000000000000715451272335422300163440ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/bogusness.C * * Modifications by: * * Brian P. Walenz from 2010-NOV-23 to 2013-AUG-01 * are Copyright 2010-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-19 to 2014-DEC-23 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "bogusUtil.H" // Reads snapper/nucmer output, figures out what the minimal set of alignments that cover each // aligned sequence, then compares those alignments against the ideal unitigs. // // It reports: // bubbles that should have been popped. // unitigs that span a unique-repeat junction (one end in unique, one end in repeat). // unitigs that include a repeat. // unitigs that include a repeat, and are chimeric at the repeat. // unitigs that should be joined. // // #define IDEAL_REPT 0 // A repeat unitig #define IDEAL_UNIQ 1 // A unique unitig #define IDEAL_REPTUNIQ 2 // A portion of this repeat unitig might be separable #define IDEAL_UNIQWEAK 3 // A portion of this unique unitig might be unjoinable #define IDEAL_MIXED 4 // Used for classifying bubbles char *types[4] = { "REPT", "UNIQ", "SEPR", "WEAK" }; #define STATUS_BEGINSin 0 #define STATUS_ENDSin 1 #define STATUS_CONTAINS 2 #define STATUS_CONTAINED 3 char *statuses[4] = { "BEGINSin", "ENDSin", "CONTAINS", "CONTAINED" }; class bogusResult { public: bogusResult(const char *_utgID, int32 _utgIID, int32 _alignNum, int32 _alignTotal, int32 _utgBgn, int32 _utgEnd, int32 _chnBgn, int32 _chnEnd, int32 _genIID, int32 _genBgn, int32 _genEnd, bool _isReverse, int32 _status, int32 _type, int32 _idlNum, int32 _idlBgn, int32 _idlEnd, int32 _alignLen, double _utgCov, double _idlCov) { if (_utgID) strcpy(utgID, _utgID); else utgID[0] = 0; utgIID = _utgIID; alignNum = _alignNum; alignTotal = _alignTotal; utgBgn = _utgBgn; utgEnd = _utgEnd; chnBgn = _chnBgn; chnEnd = _chnEnd; genIID = _genIID; genBgn = _genBgn; genEnd = _genEnd; isReverse = _isReverse; status = _status; type = _type; idlNum = _idlNum; idlBgn = _idlBgn; idlEnd = _idlEnd; alignLen = _alignLen; utgCov = _utgCov; idlCov = _idlCov; }; ~bogusResult() { }; bool operator<(const bogusResult &that) const { if (chnBgn < that.chnBgn) return(true); if (chnBgn > that.chnBgn) return(false); return(idlBgn < that.idlBgn); }; char utgID[32]; int32 utgIID; int32 alignNum; int32 alignTotal; int32 utgBgn; int32 utgEnd; int32 chnBgn; // Position in the chained reference int32 chnEnd; int32 genIID; // Position in the actual reference (for output) int32 genBgn; int32 genEnd; bool isReverse; int32 status; int32 type; int32 idlNum; int32 idlBgn; int32 idlEnd; int32 alignLen; double utgCov; double idlCov; }; class idealUnitig { public: idealUnitig() { bgn = 0; end = 0; type = 0; }; idealUnitig(int32 b, int32 e, char t) { bgn = b; end = e; type = t; }; ~idealUnitig() { }; int32 bgn; int32 end; char type; }; vector refList; map refMap; vector ideal; vector genome; map IIDmap; // Maps an ID string to an IID. vector IIDname; // Maps an IID to an ID string. vector results; void loadIdealUnitigs(char *idealName, vector &ideal) { errno = 0; FILE *F = fopen(idealName, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", idealName, strerror(errno)), exit(1); char L[1024]; splitToWords S; char t = 0; fgets(L, 1024, F); while (!feof(F)) { chomp(L); S.split(L); if (strcmp(S[3], "REPT") == 0) t = (S.numWords() == 4) ? IDEAL_REPT : IDEAL_UNIQWEAK; else if (strcmp(S[3], "UNIQ") == 0) t = (S.numWords() == 4) ? IDEAL_UNIQ : IDEAL_REPTUNIQ; else fprintf(stderr, "Unknown type in '%s'\n", L), exit(1); #warning ONLY WORKS ON ONE REFERENCE ideal.push_back(idealUnitig(S(1), S(2), t)); fgets(L, 1024, F); } fclose(F); fprintf(stderr, "Loaded %lu ideal unitigs.\n", ideal.size()); } bool isUnitigContained(int32 frgIID, idealUnitig &ideal, genomeAlignment &cover, int32 &alen, int32 &ulen, int32 &ilen, double &ufrac, double &ifrac) { if ((ideal.bgn <= cover.chnBgn) && (cover.genEnd <= ideal.end)) { alen = cover.genEnd - cover.chnBgn; ulen = cover.genEnd - cover.chnBgn; ilen = ideal.end - ideal.bgn; ufrac = 100.0 * alen / ulen; ifrac = 100.0 * alen / ilen; return(true); } return(false); } bool isUnitigContaining(int32 frgIID, idealUnitig &ideal, genomeAlignment &cover, int32 &alen, int32 &ulen, int32 &ilen, double &ufrac, double &ifrac) { if ((cover.chnBgn <= ideal.bgn) && (ideal.end <= cover.chnEnd)) { alen = ideal.end - ideal.bgn; ulen = cover.chnEnd - cover.chnBgn; ilen = ideal.end - ideal.bgn; ufrac = 100.0 * alen / ulen; ifrac = 100.0 * alen / ilen; return(true); } return(false); } bool isUnitigEnding(int32 frgIID, idealUnitig &ideal, genomeAlignment &cover, int32 &alen, int32 &ulen, int32 &ilen, double &ufrac, double &ifrac) { if ((cover.chnBgn <= ideal.bgn) && (ideal.bgn <= cover.chnEnd)) { alen = cover.chnEnd - ideal.bgn; ulen = cover.chnEnd - cover.chnBgn; ilen = ideal.end - ideal.bgn; ufrac = 100.0 * alen / ulen; ifrac = 100.0 * alen / ilen; return(true); } return(false); } bool isUnitigBeginning(int32 frgIID, idealUnitig &ideal, genomeAlignment &cover, int32 &alen, int32 &ulen, int32 &ilen, double &ufrac, double &ifrac) { if ((cover.chnBgn <= ideal.end) && (ideal.end <= cover.chnEnd)) { alen = ideal.end - cover.chnBgn; ulen = cover.chnEnd - cover.chnBgn; ilen = ideal.end - ideal.bgn; ufrac = 100.0 * alen / ulen; ifrac = 100.0 * alen / ilen; return(true); } return(false); } int main(int argc, char **argv) { uint32 nucmerNamesLen = 0; uint32 snapperNamesLen = 0; char *nucmerNames[1024]; char *snapperNames[1024]; char *idealName = 0L; char *refName = 0L; char *outputPrefix = 0L; FILE *gffOutput = 0L; FILE *resultsOutput = 0L; int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-nucmer") == 0) { nucmerNames[nucmerNamesLen++] = argv[++arg]; } else if (strcmp(argv[arg], "-snapper") == 0) { snapperNames[snapperNamesLen++] = argv[++arg]; } else if (strcmp(argv[arg], "-ideal") == 0) { idealName = argv[++arg]; } else if (strcmp(argv[arg], "-reference") == 0) { refName = argv[++arg]; } else if (strcmp(argv[arg], "-output") == 0) { outputPrefix = argv[++arg]; } else { err++; } arg++; } if ((nucmerNamesLen == 0) && (snapperNamesLen == 0L)) fprintf(stderr, "ERROR: No input matches supplied (either -nucmer or -snapper).\n"), err++; if (refName == 0L) fprintf(stderr, "ERROR: No referece supplied (-reference).\n"), err++; if (idealName == 0L) fprintf(stderr, "ERROR: No ideal unitigs supplied (-ideal).\n"), err++; if (outputPrefix == 0L) fprintf(stderr, "ERROR: No output prefix supplied (-output).\n"), err++; if (err) { exit(1); } { char outputName[FILENAME_MAX]; errno = 0; sprintf(outputName, "%s.bogusness", outputPrefix); resultsOutput = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", outputName, strerror(errno)), exit(1); sprintf(outputName, "%s.gff3", outputPrefix); gffOutput = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", outputName, strerror(errno)), exit(1); fprintf(gffOutput, "##gff-version 3\n"); } loadIdealUnitigs(idealName, ideal); loadReferenceSequence(refName, refList, refMap); for (uint32 nn=0; nn cover; // Find the range of alignments for a single fragment. uint32 end = bgn + 1; while ((end < genome.size()) && (genome[bgn].frgIID == genome[end].frgIID)) end++; // Basically, discard any alignment that is contained in some other alignment. We're not // trying to explain the genome using the unitigs, we're trying to explain (evaluate) the // unitigs using the genome. for (uint32 i=bgn; iutgID, bi->alignNum, bi->alignTotal, bi->utgBgn, bi->utgEnd, refList[bi->genIID].rsrefName, bi->genBgn, bi->genEnd, statuses[bi->status], types[bi->type], bi->idlNum, bi->idlBgn, bi->idlEnd, bi->alignLen, bi->utgCov, bi->idlCov); } fclose(gffOutput); fclose(resultsOutput); #if 0 // USELESS BROKEN STUFF //////////////////////////////////////// // // Attempt to analyze the result. // //////////////////////////////////////// // // The array variables count over various tolerances at the edges. // // -------- spans the REPT, but includes a bit of the UNIQ on // For example: uuuuu uuuu either end. If the UNIQ is less than 250bp, we'd // rrrrrr numURU{0000}=0 numURU{0250}=1 numURU{0500}=1 int32 tolerances[4] = { 0, 250, 500, 1000 }; uint32 numChimeraTotal = 0; uint32 numChimera[1024] = {0}; // Chimeric unitigs by number of pieces uint32 numBubbleUNIQ = 0; // Number of bubbles in unique regions (contained in a UNIQ and another unitig) uint32 numBubbleREPT = 0; // Number of bubbles in repeat regions (contained in a REPT and another unitig) uint32 numBubbleOTHR = 0; // Number of bubbles that span a region uint32 numIncompleteUNIQ[4] = {0}; // Number of unitigs that do not completely span a UNIQ region (**) uint32 numIncompleteREPT[4] = {0}; // Number of unitigs that do not completely span a REPT region (**) uint32 numURU[4] = {0}; // Number of unitigs that correctly span a R, and might have a bit in U on either end. uint32 numRUR[4] = {0}; // Number of unitigs that correctly span a U, and might have a bit in R on either end. uint32 numIllegalBorder[4] = {0}; // Number of unitigs that cross a UNIQ/REPT border, or multiple borders //////////////////////////////////////////////////////////////////////////////// // Chimera int32 *chm = new int32 [IIDname.size()]; memset(chm, 0, sizeof(int32) * IIDname.size()); for (uint32 i=0; ialignTotal < 1024); if ((bi->alignNum == 1) && (bi->alignTotal > 1)) chm[bi->utgIID]++; } for (uint32 i=0; ichnBgn <= gj->chnBgn) && (gj->chnEnd <= gi->chnEnd) && (gj->frgBgn == bgn[gj->frgIID]) && (gj->frgEnd == end[gj->frgIID])) { bub[gj->frgIID]++; //typ[gj->frgIID] = IDEAL_MIXED; } } } // Now, for any unitig marked as a bubble, try to determine the type. The rule is // simple. Three cases: // matches are all UNIQ // matches are all REPT // matches are of mixed type (bubble spans a junction) // for (uint32 b=0; b) { chomp; $firstRev = $_ if (!defined($firstRev)); $revCount++; } close(F); open(F, "git describe --tags --long --dirty --always --abbrev=40 |") or die "Failed to run 'git describe'.\n"; while () { chomp; if (m/^v(\d+)\.(\d+.*)-(\d+)-g(.{40})-*(.*)$/) { $major = $1; $minor = $2; $commits = $3; $hash = $4; $dirty = $5; } else { die "Failed to parse describe string '$_'.\n"; } } close(F); if ($dirty eq "dirty") { $dirty = "ahead of github"; } else { $dirty = "sync'd with guthub"; } } # If not in a git repo, we might be able to figure things out based on the directory name. elsif ((! -e "../.git") && ($cwd =~ m/canu-(.{40})\/src/)) { $hash = $1; $firstRev = $1; } # Otherwise, we know absolutely nothing. else { $hash = "unknown-hash-tag-no-repository-available"; $firstRev = "unknown-hash-tag-no-repository-available"; } # Report what we found. This is really for the gmake output. print STDERR "Building v$major.$minor (+$commits commits) r$revCount $hash ($dirty)\n"; print STDERR "\n"; # Dump a new file, but don't overwrite the original. open(F, "> canu_version.H.new") or die "Failed to open 'canu-version.H.new' for writing: $!\n"; print F "// Automagically generated by canu_version_update.pl! Do not commit!\n"; print F "#define CANU_VERSION_MAJOR \"$major\"\n"; print F "#define CANU_VERSION_MINOR \"$minor\"\n"; print F "#define CANU_VERSION_COMMITS \"$commits\"\n"; print F "#define CANU_VERSION_REVISION \"$revCount\"\n"; print F "#define CANU_VERSION_HASH \"$firstRev\"\n"; close(F); # If they're the same, don't replace the original. if (compare("canu_version.H", "canu_version.H.new") == 0) { unlink "canu_version.H.new"; } else { unlink "canu_version.H"; rename "canu_version.H.new", "canu_version.H"; } # That's all, folks! exit(0); canu-1.3/src/correction/000077500000000000000000000000001272335422300152445ustar00rootroot00000000000000canu-1.3/src/correction/errorEstimate.C000066400000000000000000000126231272335422300202010ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2016-MAY-16 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "ovStore.H" #include "splitToWords.H" #include "AS_UTL_decodeRange.H" #include "stddev.H" #include #include #include using namespace std; int main(int argc, char **argv) { char *scoreFileName = NULL; uint32 deviations = 6; float mass=0.98; bool isOvl=false; argc = AS_configure(argc, argv); int32 arg = 1; int32 err = 0; while (arg < argc) { if (strcmp(argv[arg], "-S") == 0) { scoreFileName = argv[++arg]; } else if (strcmp(argv[arg], "-d") == 0) { deviations = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-m") == 0) { mass = atof(argv[++arg]); } else if (strcmp(argv[arg], "-o") == 0) { isOvl=true; } else { fprintf(stderr, "ERROR: invalid arg '%s'\n", argv[arg]); err++; } arg++; } if (scoreFileName == NULL) err++; if (err) { fprintf(stderr, "usage: %s [options]\n", argv[0]); fprintf(stderr, "\n"); exit(1); } errno = 0; FILE *scoreFile = (scoreFileName == NULL) ? NULL : (scoreFileName[0] == '-' ? stdin : fopen(scoreFileName, "r")); if (errno) fprintf(stderr, "ERROR: failed to open '%s' for reading: %s\n", scoreFileName, strerror(errno)), exit(1); // read the file and store best hits char ovStr[1024]; ovOverlap ov(NULL); map readToLength; map readToIdy; double mean, median, stddev, mad; mean = median = stddev = mad = 0.0; while (fgets(ovStr, 1024, scoreFile) != NULL) { splitToWords W(ovStr); if (isOvl) { ov.a_iid = W(0); ov.b_iid = W(1); if (ov.a_iid == ov.b_iid) continue; ov.dat.ovl.ahg5 = W(4); ov.dat.ovl.ahg3 = W(6); ov.dat.ovl.bhg5 = W(6); ov.dat.ovl.bhg3 = W(7); ov.span(W(3)); ov.erate(atof(W[8])); ov.flipped(W[3][0] == 'I' ? true : false); } else { ov.a_iid = W(0); ov.b_iid = W(1); if (ov.a_iid == ov.b_iid) continue; assert(W[4][0] == '0'); ov.dat.ovl.ahg5 = W(5); ov.dat.ovl.ahg3 = W(7) - W(6); if (W[8][0] == '0') { ov.dat.ovl.bhg5 = W(9); ov.dat.ovl.bhg3 = W(11) - W(10); ov.flipped(false); } else { ov.dat.ovl.bhg3 = W(9); ov.dat.ovl.bhg5 = W(11) - W(10); ov.flipped(true); } ov.erate(atof(W[2])); ov.span(W(10)-W(9)); } if (ov.erate() == 0.0) ov.erate(0.01); // round up when we can't estimate accurately if (readToLength.find(ov.b_iid) == readToLength.end() || readToLength[ov.b_iid] < ov.span()) { readToLength[ov.b_iid] = ov.span(); readToIdy[ov.b_iid] = ov.erate(); } } fclose(scoreFile); stdDev edgeStats; // Find the overlap for every best edge. double *absdev = new double [readToLength.size() + 1]; double *erates = new double [readToLength.size() + 1]; uint32 eratesLen = 0; for (map::iterator it=readToIdy.begin(); it != readToIdy.end(); ++it) { edgeStats.insert(erates[eratesLen++] = it->second); } mean = edgeStats.mean(); stddev = edgeStats.stddev(); fprintf(stderr, "with %u points - mean %f stddev %f - would use overlaps below %f fraction error\n", edgeStats.size(), mean, stddev, mean + deviations * stddev); // Find the median and absolute deviations. sort(erates, erates+eratesLen); median = erates[ eratesLen / 2 ]; double massCutoff = 0; uint32 totalBelow = 0; for (uint32 ii=0; ii= 0.0); mad = absdev[eratesLen/2]; delete [] absdev; delete [] erates; fprintf(stderr, "with %u points - median %f mad %f - would use overlaps below %f fraction error\n", edgeStats.size(), median, mad, median + deviations * 1.4826 * mad); fprintf(stderr, "with %u points - mass of %d is below %f\n", edgeStats.size(), totalBelow, massCutoff); if (scoreFile) fclose(scoreFile); fprintf(stdout, "%.3f\n", massCutoff /* median + deviations * 1.4826 * mad*/); exit(0); } canu-1.3/src/correction/errorEstimate.mk000066400000000000000000000010451272335422300204220ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := errorEstimate SOURCES := errorEstimate.C SRC_INCDIRS := .. ../AS_UTL ../stores ../overlapInCore ../utgcns/libNDalign ../overlapErrorAdjustment TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/correction/filterCorrectionOverlaps.C000066400000000000000000000316741272335422300224140ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAY-28 to 2015-JUN-25 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-FEB-24 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "ovStore.H" #include "splitToWords.H" #include "AS_UTL_decodeRange.H" #include #include using namespace std; int main(int argc, char **argv) { char *gkpStoreName = NULL; char *ovlStoreName = NULL; char *scoreFileName = NULL; char logFileName[FILENAME_MAX]; char statsFileName[FILENAME_MAX]; bool noLog = false; bool noStats = false; uint32 expectedCoverage = 25; uint32 minOvlLength = 500; uint32 maxOvlLength = AS_MAX_READLEN; double maxErate = 1.0; double minErate = 1.0; bool legacyScore = false; argc = AS_configure(argc, argv); int32 arg = 1; int32 err = 0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { ovlStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-S") == 0) { scoreFileName = argv[++arg]; } else if (strcmp(argv[arg], "-c") == 0) { expectedCoverage = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-l") == 0) { minOvlLength = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-e") == 0) { AS_UTL_decodeRange(argv[++arg], minErate, maxErate); } else if (strcmp(argv[arg], "-nolog") == 0) { noLog = true; } else if (strcmp(argv[arg], "-nostats") == 0) { noStats = true; } else if (strcmp(argv[arg], "-legacy") == 0) { legacyScore = true; } else { fprintf(stderr, "ERROR: invalid arg '%s'\n", argv[arg]); err++; } arg++; } if (gkpStoreName == NULL) err++; if (ovlStoreName == NULL) err++; if (scoreFileName == NULL) err++; if (err) { fprintf(stderr, "usage: %s [options]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, "Rewrites an ovlStore, filtering overlaps that shouldn't be used for correcting reads.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -G gkpStore input reads\n"); fprintf(stderr, " -O ovlStore input overlaps\n"); fprintf(stderr, " -S scoreFile output scores for each read, binary file, to 'scoreFile'\n"); fprintf(stderr, " per-read logging to 'scoreFile.log' (see -nolog)\n"); fprintf(stderr, " summary statistics to 'scoreFile.stats' (see -nostats)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -c coverage retain at most this many overlaps per read\n"); fprintf(stderr, "\n"); fprintf(stderr, " -l length filter overlaps shorter than this length\n"); fprintf(stderr, " -e (min-)max filter overlaps outside this range of fraction error\n"); fprintf(stderr, " example: -e 0.20 filter overlaps above 20%% error\n"); fprintf(stderr, " example: -e 0.05-0.20 filter overlaps below 5%% error\n"); fprintf(stderr, " or above 20%% error\n"); fprintf(stderr, "\n"); fprintf(stderr, " -nolog don't create 'scoreFile.log'\n"); fprintf(stderr, " -nostats don't create 'scoreFile.stats'\n"); if (gkpStoreName == NULL) fprintf(stderr, "ERROR: no gatekeeper store (-G) supplied.\n"); if (ovlStoreName == NULL) fprintf(stderr, "ERROR: no overlap store (-O) supplied.\n"); if (scoreFileName == NULL) fprintf(stderr, "ERROR: no output scoreFile (-S) supplied.\n"); exit(1); } // If only one value supplied to -e, both minErate and maxErate are set to it. // Change the max to the value supplied, and set min to zero. Otherwise, two values // were supplied, and we're all set. No checking is made that the user isn't an idiot. if (minErate == maxErate) { maxErate = minErate; minErate = 0.0; } uint32 maxEvalue = AS_OVS_encodeEvalue(maxErate); uint32 minEvalue = AS_OVS_encodeEvalue(minErate);; gkStore *gkpStore = gkStore::gkStore_open(gkpStoreName); ovStore *inpStore = new ovStore(ovlStoreName, gkpStore); uint64 *scores = new uint64 [gkpStore->gkStore_getNumReads() + 1]; sprintf(logFileName, "%s.log", scoreFileName); sprintf(statsFileName, "%s.stats", scoreFileName); errno = 0; FILE *scoreFile = (scoreFileName == NULL) ? NULL : fopen(scoreFileName, "w"); if (errno) fprintf(stderr, "ERROR: failed to open '%s' for writing: %s\n", scoreFileName, strerror(errno)), exit(1); errno = 0; FILE *logFile = (noLog == true) ? NULL : fopen(logFileName, "w"); if (errno) fprintf(stderr, "ERROR: failed to open '%s' for writing: %s\n", logFileName, strerror(errno)), exit(1); uint32 ovlLen = 0; uint32 ovlMax = 131072; ovOverlap *ovl = ovOverlap::allocateOverlaps(gkpStore, ovlMax); ovOverlap swapped(gkpStore); uint32 histLen = 0; uint32 histMax = ovlMax; uint64 *hist = new uint64 [histMax]; uint64 totalOverlaps = 0; uint64 lowErate = 0; uint64 highErate = 0; uint64 tooShort = 0; uint64 tooLong = 0; uint64 belowCutoff = 0; uint64 retained = 0; uint64 totalReads = gkpStore->gkStore_getNumReads(); uint64 readsNoOlaps = 0; uint64 reads00OlapsFiltered = 0; uint64 reads50OlapsFiltered = 0; uint64 reads80OlapsFiltered = 0; uint64 reads95OlapsFiltered = 0; uint64 reads99OlapsFiltered = 0; for (uint32 id=1; id <= gkpStore->gkStore_getNumReads(); id++) { scores[id] = UINT64_MAX; inpStore->readOverlaps(id, ovl, ovlLen, ovlMax); if (ovlLen == 0) { readsNoOlaps++; continue; } if (ovl[0].a_iid != id) { readsNoOlaps++; continue; } histLen = 0; if (histMax < ovlMax) { delete [] hist; histMax = ovlMax; hist = new uint64 [ovlMax]; } // Figure out which overlaps are good enough to consider and save their length. for (uint32 oo=0; oogkStore_getNumReads() + 1); if (scoreFile) fclose(scoreFile); if (logFile) fclose(logFile); delete [] scores; delete inpStore; gkpStore->gkStore_close(); if (noStats == true) exit(0); errno = 0; FILE *statsFile = fopen(statsFileName, "w"); if (errno) fprintf(stderr, "ERROR: failed to open '%s' for writing: %s\n", statsFileName, strerror(errno)), exit(1); fprintf(statsFile, "PARAMETERS:\n"); fprintf(statsFile, "----------\n"); fprintf(statsFile, "\n"); fprintf(statsFile, "%7"F_U32P" (expected coverage)\n", expectedCoverage); fprintf(statsFile, "%7"F_U32P" (don't use overlaps shorter than this)\n", minOvlLength); fprintf(statsFile, "%7.3f (don't use overlaps with erate less than this)\n", minErate); fprintf(statsFile, "%7.3f (don't use overlaps with erate more than this)\n", maxErate); fprintf(statsFile, "\n"); fprintf(statsFile, "OVERLAPS:\n"); fprintf(statsFile, "--------\n"); fprintf(statsFile, "\n"); fprintf(statsFile, "IGNORED:\n"); fprintf(statsFile, "\n"); fprintf(statsFile, "%12"F_U64P" (< %6.4f fraction error)\n", lowErate, AS_OVS_decodeEvalue(minEvalue)); fprintf(statsFile, "%12"F_U64P" (> %6.4f fraction error)\n", highErate, AS_OVS_decodeEvalue(maxEvalue)); fprintf(statsFile, "%12"F_U64P" (< %u bases long)\n", tooShort, minOvlLength); fprintf(statsFile, "%12"F_U64P" (> %u bases long)\n", tooLong, maxOvlLength); fprintf(statsFile, "\n"); fprintf(statsFile, "FILTERED:\n"); fprintf(statsFile, "\n"); fprintf(statsFile, "%12"F_U64P" (too many overlaps, discard these shortest ones)\n", belowCutoff); fprintf(statsFile, "\n"); fprintf(statsFile, "EVIDENCE:\n"); fprintf(statsFile, "\n"); fprintf(statsFile, "%12"F_U64P" (longest overlaps)\n", retained); fprintf(statsFile, "\n"); fprintf(statsFile, "TOTAL:\n"); fprintf(statsFile, "\n"); fprintf(statsFile, "%12"F_U64P" (all overlaps)\n", totalOverlaps); fprintf(statsFile, "\n"); fprintf(statsFile, "READS:\n"); fprintf(statsFile, "-----\n"); fprintf(statsFile, "\n"); fprintf(statsFile, "%12"F_U64P" (no overlaps)\n", readsNoOlaps); fprintf(statsFile, "%12"F_U64P" (no overlaps filtered)\n", reads00OlapsFiltered); fprintf(statsFile, "%12"F_U64P" (< 50%% overlaps filtered)\n", reads50OlapsFiltered); fprintf(statsFile, "%12"F_U64P" (< 80%% overlaps filtered)\n", reads80OlapsFiltered); fprintf(statsFile, "%12"F_U64P" (< 95%% overlaps filtered)\n", reads95OlapsFiltered); fprintf(statsFile, "%12"F_U64P" (< 100%% overlaps filtered)\n", reads99OlapsFiltered); fprintf(statsFile, "\n"); fclose(statsFile); // Histogram of overlaps per read // Histogram of overlaps filtered per read // Histogram of overlaps used for correction (number per read, lengths?) exit(0); } canu-1.3/src/correction/filterCorrectionOverlaps.mk000066400000000000000000000007731272335422300226350ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := filterCorrectionOverlaps SOURCES := filterCorrectionOverlaps.C SRC_INCDIRS := .. ../AS_UTL ../stores TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/correction/generateCorrectionLayouts.C000066400000000000000000000404341272335422300225600ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-APR-09 to 2015-SEP-21 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-27 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-FEB-12 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include "tgStore.H" #include "outputFalcon.H" #include "stashContains.H" #include "splitToWords.H" #include using namespace std; // Debugging on which reads are filtered, which are used, and which are removed // to meet coverage thresholds. Very big. #undef DEBUG_LAYOUT // Generate a layout for the read in ovl[0].a_iid, using most or all of the overlaps // in ovl. tgTig * generateLayout(gkStore *gkpStore, uint64 *readScores, bool legacyScore, uint32 minEvidenceLength, double maxEvidenceErate, double maxEvidenceCoverage, ovOverlap *ovl, uint32 ovlLen, FILE *flgFile) { set children; tgTig *layout = new tgTig; layout->_tigID = ovl[0].a_iid; layout->_coverageStat = 1.0; // Default to just barely unique layout->_microhetProb = 1.0; // Default to 100% probability of unique layout->_class = tgTig_noclass; layout->_suggestRepeat = false; layout->_suggestCircular = false; gkRead *read = gkpStore->gkStore_getRead(ovl[0].a_iid); layout->_layoutLen = read->gkRead_sequenceLength(); resizeArray(layout->_children, layout->_childrenLen, layout->_childrenMax, ovlLen, resizeArray_doNothing); if (flgFile) fprintf(flgFile, "Generate layout for read "F_U32" length "F_U32" using up to "F_U32" overlaps.\n", layout->_tigID, layout->_layoutLen, ovlLen); for (uint32 oo=0; oo AS_MAX_READLEN) { char ovlString[1024]; fprintf(stderr, "ERROR: bogus overlap '%s'\n", ovl[oo].toString(ovlString, ovOverlapAsCoords, false)); } assert(ovlLength < AS_MAX_READLEN); if (ovl[oo].erate() > maxEvidenceErate) { if (flgFile) fprintf(flgFile, " filter read %9u at position %6u,%6u length %5u erate %.3f - low quality (threshold %.2f)\n", ovl[oo].b_iid, ovl[oo].a_bgn(), ovl[oo].a_end(), ovlLength, ovl[oo].erate(), maxEvidenceErate); continue; } if (ovl[oo].a_end() - ovl[oo].a_bgn() < minEvidenceLength) { if (flgFile) fprintf(flgFile, " filter read %9u at position %6u,%6u length %5u erate %.3f - too short (threshold %u)\n", ovl[oo].b_iid, ovl[oo].a_bgn(), ovl[oo].a_end(), ovlLength, ovl[oo].erate(), minEvidenceLength); continue; } if ((readScores != NULL) && (ovlScore < readScores[ovl[oo].b_iid])) { if (flgFile) fprintf(flgFile, " filter read %9u at position %6u,%6u length %5u erate %.3f - filtered by global filter (threshold "F_U64")\n", ovl[oo].b_iid, ovl[oo].a_bgn(), ovl[oo].a_end(), ovlLength, ovl[oo].erate(), readScores[ovl[oo].b_iid]); continue; } if (children.find(ovl[oo].b_iid) != children.end()) { if (flgFile) fprintf(flgFile, " filter read %9u at position %6u,%6u length %5u erate %.3f - duplicate\n", ovl[oo].b_iid, ovl[oo].a_bgn(), ovl[oo].a_end(), ovlLength, ovl[oo].erate()); continue; } if (flgFile) fprintf(flgFile, " allow read %9u at position %6u,%6u length %5u erate %.3f\n", ovl[oo].b_iid, ovl[oo].a_bgn(), ovl[oo].a_end(), ovlLength, ovl[oo].erate()); tgPosition *pos = layout->addChild(); // Set the read. Parent is always the read we're building for, hangs and position come from // the overlap. Easy as pie! if (ovl[oo].flipped() == false) { pos->set(ovl[oo].b_iid, ovl[oo].a_iid, ovl[oo].a_hang(), ovl[oo].b_hang(), ovl[oo].a_bgn(), ovl[oo].a_end()); } else { pos->set(ovl[oo].b_iid, ovl[oo].a_iid, ovl[oo].a_hang(), ovl[oo].b_hang(), ovl[oo].a_end(), ovl[oo].a_bgn()); } // Remember the unaligned bit! pos->_askip = ovl[oo].dat.ovl.bhg5; pos->_bskip = ovl[oo].dat.ovl.bhg3; // record the ID children.insert(ovl[oo].b_iid); } // Use utgcns's stashContains to get rid of extra coverage; we don't care about it, and // just delete it immediately. savedChildren *sc = stashContains(layout, maxEvidenceCoverage); if ((flgFile) && (sc)) sc->reportRemoved(flgFile, layout->tigID()); if (sc) { delete sc->children; delete sc; } // stashContains also sorts by position, so we're done. #if 0 if (flgFile) for (uint32 ii=0; iinumberOfChildren(); ii++) fprintf(flgFile, " read %9u at position %6u,%6u hangs %6d %6d %c unAl %5d %5d\n", layout->getChild(ii)->_objID, layout->getChild(ii)->_min, layout->getChild(ii)->_max, layout->getChild(ii)->_ahang, layout->getChild(ii)->_bhang, layout->getChild(ii)->isForward() ? 'F' : 'R', layout->getChild(ii)->_askip, layout->getChild(ii)->_bskip); #endif return(layout); } int main(int argc, char **argv) { char *gkpName = 0L; char *ovlName = 0L; char *scoreName = 0L; char *tigName = 0L; bool falconOutput = false; // To stdout bool trimToAlign = false; uint32 errorRate = AS_OVS_encodeEvalue(0.015); char *outputPrefix = NULL; char logName[FILENAME_MAX] = {0}; char sumName[FILENAME_MAX] = {0}; char flgName[FILENAME_MAX] = {0}; FILE *logFile = 0L; FILE *sumFile = 0L; FILE *flgFile = 0L; uint32 minEvidenceOverlap = 40; uint32 minEvidenceCoverage = 1; uint32 iidMin = 0; uint32 iidMax = UINT32_MAX; char *readListName = NULL; set readList; uint32 minEvidenceLength = 0; double maxEvidenceErate = 1.0; double maxEvidenceCoverage = DBL_MAX; uint32 minCorLength = 0; bool filterCorLength = false; bool legacyScore = false; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { // Input gkpStore gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { // Input ovlStore ovlName = argv[++arg]; } else if (strcmp(argv[arg], "-S") == 0) { // Input scores scoreName = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { // Output tigStore tigName = argv[++arg]; } else if (strcmp(argv[arg], "-F") == 0) { // Output directly to falcon, not tigStore falconOutput = true; trimToAlign = true; } else if (strcmp(argv[arg], "-p") == 0) { // Output prefix, just logging and summary outputPrefix = argv[++arg]; } else if (strcmp(argv[arg], "-b") == 0) { // Begin read range iidMin = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-e") == 0) { // End read range iidMax = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-rl") == 0) { // List of reads to correct, will also apply -b/-e range readListName = argv[++arg]; } else if (strcmp(argv[arg], "-L") == 0) { // Minimum length of evidence overlap minEvidenceLength = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-E") == 0) { // Max error rate of evidence overlap maxEvidenceErate = atof(argv[++arg]); } else if (strcmp(argv[arg], "-C") == 0) { // Max coverage of evidence reads to emit. maxEvidenceCoverage = atof(argv[++arg]); } else if (strcmp(argv[arg], "-M") == 0) { // Minimum length of a corrected read minCorLength = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-legacy") == 0) { legacyScore = true; } else { fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); err++; } arg++; } if (gkpName == NULL) err++; if (ovlName == NULL) err++; if (err) { fprintf(stderr, "usage: %s -G gkpStore -O ovlStore [ -T tigStore | -F ] ...\n", argv[0]); fprintf(stderr, " -G gkpStore mandatory path to gkpStore\n"); fprintf(stderr, " -O ovlStore mandatory path to ovlStore\n"); fprintf(stderr, "\n"); fprintf(stderr, " -S file global score (binary) input file\n"); fprintf(stderr, "\n"); fprintf(stderr, " -T corStore output layouts to tigStore corStore\n"); fprintf(stderr, " -F output falconsense-style input directly to stdout\n"); fprintf(stderr, "\n"); fprintf(stderr, " -p name output prefix name, for logging and summary\n"); fprintf(stderr, "\n"); fprintf(stderr, " -b bgnID \n"); fprintf(stderr, " -e endID \n"); fprintf(stderr, "\n"); fprintf(stderr, " -rl file \n"); fprintf(stderr, "\n"); fprintf(stderr, " -L length minimum length of evidence overlaps\n"); fprintf(stderr, " -E erate maxerror rate of evidence overlaps\n"); fprintf(stderr, "\n"); fprintf(stderr, " -C coverage maximum coverage of evidence reads to emit\n"); fprintf(stderr, " -M length minimum length of a corrected read\n"); fprintf(stderr, "\n"); if (gkpName == NULL) fprintf(stderr, "ERROR: no gkpStore input (-G) supplied.\n"); if (ovlName == NULL) fprintf(stderr, "ERROR: no ovlStore input (-O) supplied.\n"); exit(1); } // Open inputs and output tigStore. gkStore *gkpStore = gkStore::gkStore_open(gkpName); ovStore *ovlStore = new ovStore(ovlName, gkpStore); tgStore *tigStore = (tigName != NULL) ? new tgStore(tigName) : NULL; // Load read scores, if supplied. uint64 *readScores = NULL; if (scoreName) { readScores = new uint64 [gkpStore->gkStore_getNumReads() + 1]; errno = 0; FILE *scoreFile = fopen(scoreName, "r"); if (errno) fprintf(stderr, "failed to open '%s' for reading: %s\n", scoreName, strerror(errno)), exit(1); AS_UTL_safeRead(scoreFile, readScores, "scores", sizeof(uint64), gkpStore->gkStore_getNumReads() + 1); fclose(scoreFile); } // Threshold the range of reads to operate on. if (gkpStore->gkStore_getNumReads() < iidMin) { fprintf(stderr, "ERROR: only "F_U32" reads in the store (IDs 0-"F_U32" inclusive); can't process requested range -b "F_U32" -e "F_U32"\n", gkpStore->gkStore_getNumReads(), gkpStore->gkStore_getNumReads()-1, iidMin, iidMax); exit(1); } if (gkpStore->gkStore_getNumReads() < iidMax) iidMax = gkpStore->gkStore_getNumReads(); ovlStore->setRange(iidMin, iidMax); // If a readList is supplied, load it, respecting the iidMin/iidMax (only to cut down on the // size). if (readListName != NULL) { errno = 0; char L[1024]; FILE *R = fopen(readListName, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", readListName, strerror(errno)), exit(1); fgets(L, 1024, R); while (!feof(R)) { splitToWords W(L); uint32 id = W(0); if ((iidMin <= id) && (id <= iidMax)) readList.insert(W(0)); fgets(L, 1024, R); } fclose(R); } // Open logging and summary files if (outputPrefix) { sprintf(logName, "%s.log", outputPrefix); sprintf(sumName, "%s.summary", outputPrefix); sprintf(flgName, "%s.filter.log", outputPrefix); errno = 0; logFile = fopen(logName, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", logName, strerror(errno)), exit(1); sumFile = fopen(sumName, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", sumName, strerror(errno)), exit(1); #ifdef DEBUG_LAYOUT flgFile = fopen(flgName, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", flgName, strerror(errno)), exit(1); #endif } if (logFile) fprintf(logFile, "read\torigLen\tnumOlaps\tcorLen\n"); // Initialize processing. uint32 ovlMax = 1024 * 1024; uint32 ovlLen = 0; ovOverlap *ovl = ovOverlap::allocateOverlaps(gkpStore, ovlMax); ovlLen = ovlStore->readOverlaps(ovl, ovlMax, true); gkReadData *readData = new gkReadData; // And process. while (ovlLen > 0) { bool skipIt = false; char skipMsg[1024] = {0}; tgTig *layout = generateLayout(gkpStore, readScores, legacyScore, minEvidenceLength, maxEvidenceErate, maxEvidenceCoverage, ovl, ovlLen, flgFile); // If there was a readList, skip anything not in it. if ((readListName != NULL) && (readList.count(layout->tigID()) == 0)) { strcat(skipMsg, "\tnot_in_readList"); skipIt = true; } // Possibly filter by the length of the uncorrected read. gkRead *read = gkpStore->gkStore_getRead(layout->tigID()); if (read->gkRead_sequenceLength() < minCorLength) { strcat(skipMsg, "\tread_too_short"); skipIt = true; } // Possibly filter by the length of the corrected read. uint32 minPos = UINT32_MAX; uint32 maxPos = 0; uint32 corLen = 0; for (uint32 ii=0; iinumberOfChildren(); ii++) { tgPosition *pos = layout->getChild(ii); if (pos->_min < minPos) minPos = pos->_min; if (maxPos < pos->_max) maxPos = pos->_max; } if (minPos != UINT32_MAX) corLen = maxPos - minPos; if (corLen < minCorLength) { strcat(skipMsg, "\tcorrection_too_short"); skipIt = true; } // Filter out empty tigs - these either have no overlaps, or failed the // length check in generateLayout. if (layout->numberOfChildren() <= 1) { strcat(skipMsg, "\tno_children"); skipIt = true; } // Output, if not skipped. if (logFile) fprintf(logFile, "%u\t%u\t%u\t%u%s\n", layout->tigID(), read->gkRead_sequenceLength(), layout->numberOfChildren(), corLen, skipMsg); if ((skipIt == false) && (tigStore != NULL)) tigStore->insertTig(layout, false); if ((skipIt == false) && (falconOutput == true)) outputFalcon(gkpStore, layout, trimToAlign, stdout, readData); delete layout; // Load next batch of overlaps. ovlLen = ovlStore->readOverlaps(ovl, ovlMax, true); } if (falconOutput) fprintf(stdout, "- -\n"); delete readData; if (logFile != NULL) fclose(logFile); if (sumFile != NULL) fclose(sumFile); if (flgFile != NULL) fclose(flgFile); delete tigStore; delete ovlStore; gkpStore->gkStore_close(); return(0); } canu-1.3/src/correction/generateCorrectionLayouts.mk000066400000000000000000000011201272335422300227720ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := generateCorrectionLayouts SOURCES := generateCorrectionLayouts.C ../utgcns/stashContains.C ../falcon_sense/outputFalcon.C SRC_INCDIRS := .. ../AS_UTL ../stores ../utgcns ../falcon_sense TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/correction/readConsensus.C000066400000000000000000000351161272335422300201720ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-JUN-23 to 2015-JUL-26 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "sweatShop.H" //#include #include "gkStore.H" #include "ovStore.H" #include "tgStore.H" #include "overlapReadCache.H" #include "NDalign.H" #include "analyzeAlignment.H" #include "AS_UTL_reverseComplement.H" #include "timeAndSize.H" // getTime(); class consensusGlobalData { public: consensusGlobalData(double maxErate_, uint32 bgnID_, int32 endID_, char *gkpName, char *ovlName, char *tigName, uint32 tigVers, char *cnsName, char *fastqName, uint64 memLimit_) { // Inputs gkpStore = gkStore::gkStore_open(gkpName); readCache = new overlapReadCache(gkpStore, memLimit); ovlStore = (ovlName) ? new ovStore(ovlName, gkpStore) : NULL; tigStore = (tigName) ? new tgStore(tigName, tigVers) : NULL; if (ovlStore) fprintf(stderr, "consensusGlobalData()-- opened ovlStore '%s'\n", ovlName); if (tigStore) fprintf(stderr, "consensusGlobalData()-- opened tigStore '%s'\n", tigName); // Parameters maxErate = maxErate_; memLimit = memLimit_; if (bgnID_ == 0) bgnID_ = 1; if (endID_ > gkpStore->gkStore_getNumReads()) endID_ = gkpStore->gkStore_getNumReads() + 1; bgnID = bgnID_; curID = bgnID_; endID = endID_; // Outputs cnsFile = NULL; fastqFile = NULL; if (cnsName) { errno = 0; cnsFile = fopen(cnsName, "w"); if (errno) fprintf(stderr, "ERROR: failed to open '%s' for writing: %s\n", cnsName, strerror(errno)), exit(1); } if (fastqName) { errno = 0; fastqFile = fopen(fastqName, "w"); if (errno) fprintf(stderr, "ERROR: failed to open '%s' for writing: %s\n", fastqName, strerror(errno)), exit(1); } // State for loading overlaps // State for loading tigs }; ~consensusGlobalData() { gkpStore->gkStore_close(); delete readCache; delete ovlStore; delete tigStore; if (cnsFile) fclose(cnsFile); if (fastqFile) fclose(fastqFile); }; // Parameters double maxErate; uint64 memLimit; uint32 bgnID; uint32 curID; // Currently loading id uint32 endID; // Inputs gkStore *gkpStore; overlapReadCache *readCache; ovStore *ovlStore; tgStore *tigStore; // State for loading gkReadData readData; // State for loading overlaps // State for loading tigs // Outputs FILE *cnsFile; FILE *fastqFile; }; class consensusThreadData { public: consensusThreadData(consensusGlobalData *g, uint32 tid) { threadID = tid; nPassed = 0; nFailed = 0; align = new NDalign(pedGlobal, g->maxErate, 15); // true = partial aligns, maxErate, seedSize analyze = new analyzeAlignment(); }; ~consensusThreadData() { delete align; delete analyze; }; uint32 threadID; uint64 nPassed; uint64 nFailed; char bRev[AS_MAX_READLEN]; NDalign *align; analyzeAlignment *analyze; }; // The overlap compute needs both strings in the correct orientation. // This loaded just loads the tig/overlaps, and converts to a common format. // It ensures that the overlapReadCache is loaded. // aID aBgn aEnd // bID bBgn bEnd bFlip class consensusComputation { public: consensusComputation(tgTig *tig) { _tig = tig; _corLen = 0; _corSeq = NULL; _corQlt = NULL; }; ~consensusComputation() { delete [] _corSeq; delete [] _corQlt; }; public: tgTig *_tig; // Input //ovOverlap *_overlaps; // Input, we convert this to a tig... //uint32 _overlapsLen; uint32 _corLen; char *_corSeq; // Output sequence char *_corQlt; }; // Complicated. Load the overlaps, filter, and convert to a tig. consensusComputation * consensusReaderOverlaps(consensusGlobalData *UNUSED(g)) { return(NULL); } // Simple, just load the tig and call it a day. consensusComputation * consensusReaderTigs(consensusGlobalData *g) { tgTig *t = NULL; consensusComputation *s = NULL; while ((t == NULL) && (g->curID < g->endID)) t = g->tigStore->loadTig(g->curID++); if (t) s = new consensusComputation(t); return(s); } void * consensusReader(void *G) { consensusGlobalData *g = (consensusGlobalData *)G; consensusComputation *s = NULL; if ((g->curID < g->endID) && (g->ovlStore)) s = consensusReaderOverlaps(g); if ((g->curID < g->endID) && (g->tigStore)) s = consensusReaderTigs(g); if (s) g->readCache->loadReads(s->_tig); return(s); } void consensusWorker(void *G, void *T, void *S) { consensusGlobalData *g = (consensusGlobalData *)G; consensusThreadData *t = (consensusThreadData *)T; consensusComputation *s = (consensusComputation *)S; uint32 rID = s->_tig->tigID(); fprintf(stderr, "THREAD %u working on tig %u\n", t->threadID, rID); t->analyze->reset(rID, g->readCache->getRead(rID), g->readCache->getLength(rID)); for (uint32 oo=0; oo_tig->numberOfChildren(); oo++) { if (s->_tig->getChild(oo)->isRead() == false) continue; tgPosition *pos = s->_tig->getChild(oo); // // This colosely follows overlapPair // // Load A. uint32 aID = s->_tig->tigID(); char *aStr = g->readCache->getRead (aID); uint32 aLen = g->readCache->getLength(aID); int32 aLo = pos->min() - 100; if (aLo < 0) aLo = 0; int32 aHi = pos->max() + 100; assert(aID == rID); assert(aLo < aHi); // Load B. If reversed, we need to reverse the coordinates to meet the overlap spec. uint32 bID = pos->ident(); char *bStr = g->readCache->getRead (bID); uint32 bLen = g->readCache->getLength(bID); int32 bLo = (pos->isReverse() == false) ? ( pos->askip()) : (bLen - pos->askip()); int32 bHi = (pos->isReverse() == false) ? (bLen - pos->bskip()) : ( pos->bskip()); // Compute the overlap #if 0 fprintf(stderr, "aligned %6u %s %6u -- %5u-%5u %5u-%5u\n", aID, pos->isReverse() ? "<--" : "-->", bID, aLo, aHi, bLo, bHi); #endif t->align->initialize(aID, aStr, aLen, aLo, aHi, bID, bStr, bLen, bLo, bHi, pos->isReverse()); if (t->align->findMinMaxDiagonal(40) == false) continue; if (t->align->findSeeds(false) == false) continue; t->align->findHits(); t->align->chainHits(); if (t->align->processHits() == true) { t->nPassed++; int32 aLoO = t->align->abgn(); int32 aHiO = t->align->aend(); int32 bLoO = t->align->bbgn(); int32 bHiO = t->align->bend(); if (bLoO > bHiO) { bLoO = t->align->bend(); bHiO = t->align->bbgn(); } #if 0 fprintf(stderr, "aligned %6u %s %6u -- %5u-%5u %5u-%5u -- %5u-%5u %5u-%5u -- %6.2f\n", aID, pos->isReverse() ? "<--" : "-->", bID, aLo, aHi, bLo, bHi, aLoO, aHiO, bLoO, bHiO, 100.0 * (aHiO - aLoO) / (aHi - aLo)); //t->align->display(); #endif // This wants pointers to the start of the strings that align, and the offset to the full read. t->analyze->analyze(t->align->astr() + aLoO, aHiO - aLoO, aLoO, t->align->bstr() + bLoO, bHiO - bLoO, t->align->deltaLen(), t->align->delta()); } else { t->nFailed++; #if 1 fprintf(stderr, "FAILED %6u %s %6u -- %5u-%5u %5u-%5u -- %5u-%5u %5u-%5u -- %6.2f\n", aID, pos->isReverse() ? "<--" : "-->", bID, aLo, aHi, bLo, bHi, 0, 0, 0, 0, 0.0); //t->align->display(); #endif } } //fprintf(stderr, "THREAD %2u finished with tig %5u -- passed %12u -- failed %12u\n", t->threadID, s->_tig->tigID(), t->nPassed, t->nFailed); t->analyze->generateCorrections(); t->analyze->generateCorrectedRead(); s->_corLen = t->analyze->_corSeqLen; s->_corSeq = new char [s->_corLen + 1]; s->_corQlt = new char [s->_corLen + 1]; memcpy(s->_corSeq, t->analyze->_corSeq, sizeof(char) * (s->_corLen + 1)); memcpy(s->_corQlt, t->analyze->_corSeq, sizeof(char) * (s->_corLen + 1)); for (uint32 ii=0; ii_corLen; ii++) s->_corQlt[ii] = '!' + 10; } void consensusWriter(void *G, void *S) { consensusGlobalData *g = (consensusGlobalData *)G; consensusComputation *s = (consensusComputation *)S; if (g->cnsFile) { } if (g->fastqFile) { fprintf(g->fastqFile, "@read%08u\n", s->_tig->tigID()); fprintf(g->fastqFile, "%s\n", s->_corSeq); fprintf(g->fastqFile, "+\n"); fprintf(g->fastqFile, "%s\n", s->_corQlt); } delete s; } int main(int argc, char **argv) { char *gkpName = NULL; char *ovlName = NULL; char *tigName = NULL; uint32 tigVers = 1; char *cnsName = NULL; char *fastqName = NULL; uint32 bgnID = 1; uint32 endID = UINT32_MAX; uint32 numThreads = 1; double maxErate = 0.02; uint64 memLimit = 4; argc = AS_configure(argc, argv); int err=0; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { ovlName = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { tigName = argv[++arg]; tigVers = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-c") == 0) { cnsName = argv[++arg]; } else if (strcmp(argv[arg], "-f") == 0) { fastqName = argv[++arg]; } else if (strcmp(argv[arg], "-b") == 0) { bgnID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-e") == 0) { endID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-t") == 0) { numThreads = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-erate") == 0) { maxErate = atof(argv[++arg]); } else if (strcmp(argv[arg], "-memory") == 0) { memLimit = atoi(argv[++arg]); } else { err++; } arg++; } if (gkpName == NULL) err++; if ((ovlName == NULL) && (tigName == NULL)) err++; if ((ovlName != NULL) && (tigName != NULL)) err++; if ((cnsName == NULL) && (fastqName == NULL)) err++; if (err) { fprintf(stderr, "usage: %s ...\n", argv[0]); fprintf(stderr, " -G gkpStore Mandatory, path to gkpStore\n"); fprintf(stderr, "\n"); fprintf(stderr, "Inputs can come from either an overlap or a tig store.\n"); fprintf(stderr, " -O ovlStore \n"); fprintf(stderr, " -T tigStore tigVers \n"); fprintf(stderr, "\n"); fprintf(stderr, "If from an ovlStore, the range of reads processed can be restricted.\n"); fprintf(stderr, " -b bgnID \n"); fprintf(stderr, " -e endID \n"); fprintf(stderr, "\n"); fprintf(stderr, "Outputs will be written as the full multialignment and the final consensus sequence\n"); fprintf(stderr, " -c output.cns \n"); fprintf(stderr, " -f output.fastq \n"); fprintf(stderr, "\n"); fprintf(stderr, " -erate e Overlaps are computed at 'e' fraction error; must be larger than the original erate\n"); fprintf(stderr, " -memory m Use up to 'm' GB of memory\n"); fprintf(stderr, "\n"); fprintf(stderr, " -t n Use up to 'n' cores\n"); fprintf(stderr, "\n"); if (gkpName == NULL) fprintf(stderr, "ERROR: no gatekeeper (-G) supplied.\n"); if ((ovlName == NULL) && (tigName == NULL)) fprintf(stderr, "ERROR: no inputs (-O or -T) supplied.\n"); if ((ovlName != NULL) && (tigName != NULL)) fprintf(stderr, "ERROR: only one input (-O or -T) may be supplied.\n"); if ((cnsName == NULL) && (fastqName == NULL)) fprintf(stderr, "ERROR: no outputs (-c or -f) supplied.\n"); exit(1); } consensusGlobalData *g = new consensusGlobalData(maxErate, bgnID, endID, gkpName, ovlName, tigName, tigVers, cnsName, fastqName, memLimit); #if 1 consensusThreadData *t = new consensusThreadData(g, 0); while (1) { consensusComputation *c = (consensusComputation *)consensusReader(g); if (c == NULL) break; consensusWorker(g, t, c); consensusWriter(g, c); } delete t; #else consensusThreadData **td = new consensusThreadData * [numThreads]; sweatShop *ss = new sweatShop(consensusReader, consensusWorker, consensusWriter); ss->setLoaderQueueSize(16384); ss->setWriterQueueSize(1024); ss->setNumberOfWorkers(numThreads); for (uint32 w=0; wsetThreadData(w, td[w] = new consensusThreadData(g, w)); // these leak ss->run(g, true); delete ss; for (uint32 w=0; w $prefix.stats.dat"); open(E, "> $prefix.erate.dat"); # Ignore the header. $_ = ; while () { s/^\s+//; s/\s+$//; #my @v = split '\s+', $_; #my $biid = $v[0]; #my $abgn = $v[2]; #my $aend = $v[3]; my ($biid, $abgn, $aend, $erate); if (m/(\d+)\s+A:\s+(\d+)\s+(\d+)\s+\(\s*\d+\)\s+B:\s+\d+\s+\d+\s+\(\s*\d+\)\s+(\d+.\d+)%/) { $biid = $1; $abgn = $2; $aend = $3; $erate = $4; } else { print "NOPE $_\n"; } for (my $ii=$abgn; $ii<=$aend; $ii++) { if (!defined($erates[$ii])) { $erates[$ii] = "$erate"; } else { $erates[$ii] .= ":$erate"; } } #print E "$abgn\t$aend\t$erate\n"; } close(F); my $maxPoint = scalar(@erates); for (my $pp=0; $pp<$maxPoint; $pp++) { my @vals = split ':', $erates[$pp]; my $depth = scalar (@vals); @vals = sort {$a <=> $b } @vals; my $min = $vals[0]; my $max = $vals[$depth-1]; my $sum = 0; foreach my $v (@vals) { $sum += $v; } my $ave = $sum / $depth; print D "$pp\t$min\t$ave\t$max\t$depth\n"; foreach my $v (@vals) { print E "$pp\t$v\n"; } } close(D); close(E); system("overlapStore -p $iid test.ovlStore test.gkpStore CLR > $prefix.ovlPicture"); open(F, "| gnuplot"); print F "set terminal 'png' size 1280,800\n"; print F "set output '$prefix.png'\n"; print F "plot [][0:30]"; print F " '$prefix.stats.dat' using 1:2 with lines title 'min % identity',"; print F " '$prefix.stats.dat' using 1:3 with lines title 'ave % identity',"; print F " '$prefix.stats.dat' using 1:4 with lines title 'max % identity',"; print F " '$prefix.stats.dat' using 1:5 with lines title 'depth',"; #print F " '$prefix.erate.dat' using 1:2 with points ps 0.2 title 'erate'\n"; print F " '$prefix.erate.dat' using 1:3 with points ps 0.3 title 'bgnerates',"; print F " '$prefix.erate.dat' using 2:3 with points ps 0.3 title 'enderates'\n"; close(F); } canu-1.3/src/erateEstimate/erate-estimate-test-based-on-mapping.pl000066400000000000000000000134301272335422300252340ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/AS_BAT/erate-estimate-test-based-on-mapping.pl # # Modifications by: # # Brian P. Walenz from 2014-NOV-15 to 2015-AUG-07 # are Copyright 2014-2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $erateFile = "BO/test.blasr.sam.coords.longestErate"; my $uidMapFile = "CAordered/test.gkpStore.fastqUIDmap"; my $olapStore = "CAordered/test.ovlStore"; my %erate5; # Either the mapped global erate, or the lowest erate for each end my %erate3; # Load the map from IID to UID and name. my %UIDtoIID; my %UIDtoNAM; my %IIDtoUID; my %IIDtoNAM; my %NAMtoUID; my %NAMtoIID; if (defined($uidMapFile)) { print STDERR "Read names from '$uidMapFile'\n"; open(F, "< $uidMapFile"); while () { my @v = split '\s+', $_; # UID IID NAM UID IID NAM $UIDtoIID{$v[0]} = $v[1]; $UIDtoNAM{$v[0]} = $v[2]; $IIDtoUID{$v[1]} = $v[0]; $IIDtoNAM{$v[1]} = $v[2]; $NAMtoUID{$v[2]} = $v[0]; $NAMtoIID{$v[2]} = $v[1]; if (scalar(@v) == 6) { $UIDtoIID{$v[3]} = $v[4]; $UIDtoNAM{$v[3]} = $v[5]; $IIDtoUID{$v[4]} = $v[3]; $IIDtoNAM{$v[4]} = $v[5]; $NAMtoUID{$v[5]} = $v[3]; $NAMtoIID{$v[5]} = $v[4]; } } close(F); } if (0) { print STDERR "Load global read erates from '$erateFile'\n"; my $found = 0; my $lost = 0; if (! -e "$erateFile") { die "Run erate-coords.pl in BO directory.\n"; } open(F, "< $erateFile") or die; while() { my @v = split '\s+', $_; # This should be read NAME, not assmelby UID. my $iid = $NAMtoIID{$v[0]}; if (defined($iid)) { $found++; $erate5{$iid} = 100 - $v[1]; $erate3{$iid} = 100 - $v[1]; } else { $lost++; #print STDERR "Didn't find name '$v[0]' - dropped from read set?\n"; } } close(F); print STDERR "Found $found erates, lost $lost.\n"; } if (1) { print STDERR "Load overlap erates from '$olapStore'\n"; open(F, "overlapStore -d $olapStore |"); while () { s/^\s+//; s/\s+$//; #print "$_\n"; my ($aIID, $bIID, $orient, $aHang, $bHang, $erate, $crate) = split '\s+', $_; if (($aHang <= 0) && ($bHang >= 0)) { # A contained in B next; } elsif (($aHang >= 0) && ($bHang <= 0)) { # B contained in A next; } elsif (($aHang <= 0) && ($bHang <= 0)) { # Dovetail off the 5' end $erate5{$aIID} = $erate if ((!exists($erate5{$aIID})) || ($erate < $erate5{$aIID})); } elsif (($aHang >= 0) && ($bHang >= 0)) { # Dovetail off the 3' end $erate3{$aIID} = $erate if ((!exists($erate3{$aIID})) || ($erate < $erate3{$aIID})); } else { die; } } } # Scan overlaps, try to decide true from false. print STDERR "Read overlas from '$olapStore'\n"; my @ratioTrue; my @ratioFalse; open(F, "overlapStore -d $olapStore |"); while () { s/^\s+//; s/\s+$//; #print "$_\n"; my ($aIID, $bIID, $orient, $aHang, $bHang, $erate, $crate) = split '\s+', $_; my $aErate = 0; my $bErate = 0; if (($aHang <= 0) && ($bHang >= 0)) { # A contained in B next; } elsif (($aHang >= 0) && ($bHang <= 0)) { # B contained in A next; } elsif (($aHang <= 0) && ($bHang <= 0) && ($orient eq "N")) { # Dovetail off the 5' end of A to the 3' end of B $aErate = $erate5{$aIID}; $bErate = $erate3{$bIID}; } elsif (($aHang <= 0) && ($bHang <= 0) && ($orient eq "I")) { # Dovetail off the 5' end of A to the 5' end of B $aErate = $erate5{$aIID}; $bErate = $erate5{$bIID}; } elsif (($aHang >= 0) && ($bHang >= 0) && ($orient eq "N")) { # Dovetail off the 3' end of A to the 5' end of B $aErate = $erate3{$aIID}; $bErate = $erate5{$bIID}; } elsif (($aHang >= 0) && ($bHang >= 0) && ($orient eq "I")) { # Dovetail off the 3' end of A to the 3' end of B $aErate = $erate3{$aIID}; $bErate = $erate3{$bIID}; } else { die; } print STDERR "Didn't find erate for IID $aIID.\n $_\n" if (!defined($aErate)); print STDERR "Didn't find erate for IID $bIID.\n $_\n" if (!defined($bErate)); next if (!defined($aErate)); next if (!defined($bErate)); my $ratio = int(100 * ($erate) / ($aErate + $bErate)); my $dist = ($aIID < $bIID) ? ($bIID - $aIID) : ($aIID - $bIID); die if ($dist < 0); if ($dist < 50) { $ratioTrue[$ratio]++; } else { $ratioFalse[$ratio]++; } } close(F); open(F, "> r"); for (my $x=0; $x<1000; $x++) { next if (!defined($ratioTrue[$x]) && (!defined($ratioFalse[$x]))); $ratioTrue[$x] += 0; $ratioFalse[$x] += 0; my $v = $x / 100; print F "$v\t$ratioTrue[$x]\t$ratioFalse[$x]\n"; } close(F); canu-1.3/src/erateEstimate/erateEstimate.C000066400000000000000000000505441272335422300206010ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/erate-estimate.C * * Modifications by: * * Brian P. Walenz from 2014-OCT-21 to 2015-JUL-29 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include "memoryMappedFile.H" #include "intervalList.H" #include #include #include #include #include using namespace std; #ifndef BROKEN_CLANG_OpenMP #include #endif #define ERATE_TOLERANCE 0.03 uint32 blockSize = 1000; class readErrorEstimate { public: readErrorEstimate() { seqLen = 0; errorMeanS = NULL; // Sum of error up to this point //errorStdDev = NULL; //errorConfidence = NULL; errorMeanU = NULL; // Updated error estimate for this point }; ~readErrorEstimate() { delete [] errorMeanS; //delete [] errorStdDev; //delete [] errorConfidence; delete [] errorMeanU; }; uint64 initialize(gkRead *read) { //uint32 lid = read->gkRead_getLibraryIID(); //gkLibrary *lb = gkpStore->gkStore_getLibrary(lid); seqLen = read->gkRead_sequenceLength(); errorMeanS = new uint32 [seqLen + 1]; //errorStdDev = new double [seqLen + 1]; //errorConfidence = new double [seqLen + 1]; errorMeanU = new uint16 [seqLen + 1]; #if 0 uint32 err40 = AS_OVS_encodeEvalue(0.40); errorMeanS[0] = err40; errorMeanU[0] = 0; // instead of this loop, we can build one full (0-max_read_len) array, // then copy the start N elements to each errorMeanS for (uint32 ii=1; ii> 14) & 0x000001ff; b_iid_lo = (ovl.b_iid >> 0) & 0x00003fff; a_hang = ovl.a_hang(); b_hang = ovl.b_hang(); erate = ovl.evalue(); flipped = ovl.flipped(); discarded = false; assert(ovl.a_iid == a_iid); assert(ovl.b_iid == ((b_iid_hi << 14) | (b_iid_lo))); assert(ovl.a_hang() == a_hang); assert(ovl.b_hang() == b_hang); assert(ovl.erate() == erate); assert(ovl.flipped() == flipped); }; #if 0 void populateOBT(ovOverlap &obt, readErrorEstimate *readProfile, uint32 iidMin) { obt.a_iid = a_iid; obt.b_iid = (b_iid_hi << 14) | (b_iid_lo); int32 seqLenA = readProfile[obt.a_iid - iidMin].seqLen; int32 seqLenB = readProfile[obt.b_iid - iidMin].seqLen; // Swiped from AS_OVS_overlap.C uint32 abgn = (a_hang < 0) ? (0) : (a_hang); uint32 aend = (b_hang < 0) ? (seqLenA + b_hang) : (seqLenA); uint32 bbgn = (a_hang < 0) ? (-a_hang) : (0); uint32 bend = (b_hang < 0) ? (seqLenB) : (seqLenB - b_hang); obt.dat.obt.type = AS_OVS_TYPE_OBT; obt.dat.obt.a_beg = abgn; obt.dat.obt.a_end = aend; obt.dat.obt.b_beg = bbgn; obt.dat.obt.b_end_hi = (bend >> 9) & 0x00003fff; obt.dat.obt.b_end_lo = (bend) & 0x000001ff; obt.dat.obt.fwd = (flipped == false); obt.dat.obt.erate = erate; }; #endif }; class ESToverlapSpan { public: ESToverlapSpan(ESToverlap& ovl, readErrorEstimate *readProfile, uint32 iidMin) { a_iid = ovl.a_iid; b_iid = (ovl.b_iid_hi << 14) | (ovl.b_iid_lo); int32 seqLenA = readProfile[a_iid - iidMin].seqLen; int32 seqLenB = readProfile[b_iid - iidMin].seqLen; // Swiped from AS_OVS_overlap.C uint32 abgn = (ovl.a_hang < 0) ? (0) : (ovl.a_hang); uint32 aend = (ovl.b_hang < 0) ? (seqLenA + ovl.b_hang) : (seqLenA); uint32 bbgn = (ovl.a_hang < 0) ? (-ovl.a_hang) : (0); uint32 bend = (ovl.b_hang < 0) ? (seqLenB) : (seqLenB - ovl.b_hang); a_beg = abgn; a_end = aend; b_beg = bbgn; b_end = bend; fwd = (ovl.flipped == false); erate = ovl.erate; }; uint32 a_iid; uint32 b_iid; uint32 a_beg; uint32 a_end; uint32 b_beg; uint32 b_end; uint32 fwd; uint32 erate; }; void saveProfile(uint32 iid, uint32 iter, readErrorEstimate *readProfile) { char N[FILENAME_MAX]; sprintf(N, "erate-%08u-%02u.dat", iid, iter); FILE *F = fopen(N, "w"); for (uint32 pp=0; ppgkStore_getNumReads(), iter); #pragma omp parallel for schedule(dynamic, blockSize) for (uint32 iid=0; iid eRateList; for (uint64 oo=overlapIndex[iid]; oo 0) { double estError = computeEstimatedErate(iidMin, ovl, readProfile); if (estError + ERATE_TOLERANCE < erate) { overlaps[oo].discarded = true; nDiscard++; continue; } } // Otherwise, add it to the list of intervals. nRemain++; eRateList.add(ovl.a_beg, ovl.a_end - ovl.a_beg, erate / 2); } // Convert the list to a sum of error rate per base intervalList eRateMap(eRateList); // Unpack the list into an array of mean error rate per base memset(readProfile[iid].errorMeanU, 0, sizeof(uint16) * (readProfile[iid].seqLen + 1)); for (uint32 ii=0; ii 0) ? (eRateMap.value(ii) / eRateMap.depth(ii)) : 0; assert(0.0 <= eVal); assert(eVal <= 1.0); assert(eRateMap.hi(ii) <= readProfile[iid].seqLen); uint16 eEnc = AS_OVS_encodeEvalue(eVal); for (uint32 pp=eRateMap.lo(ii); pp < eRateMap.hi(ii); pp++) readProfile[iid].errorMeanU[pp] = eEnc; } // Keep users entertained. if ((iid % 1000) == 0) fprintf(stderr, "IID %u\r", iid); } // All new estimates are computed. Convert the array of mean error per base into an array of // summed error per base for (uint32 iid=0; iidnumOverlapsInRange(); fprintf(stderr, "Processing from IID "F_U32" to "F_U32" out of "F_U32" reads.\n", iidMin, iidMin + numIIDs, gkpStore->gkStore_getNumReads()); // Can't thread. This does sequential output. Plus, it doesn't compute anything. // Overlaps in the store and those in the list should be in lock-step. We can just // walk down each. uint32 overlapblock = 100000000; ovOverlap *overlapsload = ovOverlap::allocateOverlaps(gkpStore, overlapblock); for (uint64 no=0; noreadOverlaps(overlapsload, overlapblock, false); assert(nLoad > 0); for (uint32 xx=0; xxwriteOverlap(overlapsload + xx); nRemain++; } if ((no & 0x000fffff) == 0) fprintf(stderr, " overlap %10"F_U64P" %8"F_U32P"-%8"F_U32P"\r", no, a_iid, b_iid); } } delete [] overlapsload; #if 0 for (uint32 iid=0; iidwriteOverlap(&obt); } if ((iid % 1000) == 0) fprintf(stderr, "IID %u\r", iid); } #endif delete outStore; delete inpStore; fprintf(stderr, "\n"); fprintf(stderr, "nDiscarded "F_U64" (in previous iterations)\n", nDiscarded); fprintf(stderr, "nRemain "F_U64"\n", nRemain); } int main(int argc, char **argv) { char *gkpName = 0L; gkStore *gkpStore = 0L; char *ovlStoreName = 0L; ovStore *ovlStore = 0L; char *ovlCacheName = 0L; uint32 errorRate = AS_OVS_encodeEvalue(0.015); double errorLimit = 2.5; char *outputPrefix = NULL; char logName[FILENAME_MAX] = {0}; char sumName[FILENAME_MAX] = {0}; FILE *logFile = 0L; FILE *sumFile = 0L; argc = AS_configure(argc, argv); uint32 minEvidenceOverlap = 40; uint32 minEvidenceCoverage = 1; uint32 iidMin = UINT32_MAX; uint32 iidMax = UINT32_MAX; uint32 partNum = 0; uint32 partMax = 1; uint32 minLen = 0; double minErate = 0; int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { ovlStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-C") == 0) { ovlCacheName = argv[++arg]; } else if (strcmp(argv[arg], "-b") == 0) { iidMin = atoi(argv[++arg]); partNum = 0; partMax = 0; } else if (strcmp(argv[arg], "-e") == 0) { iidMax = atoi(argv[++arg]); partNum = 0; partMax = 0; } else if (strcmp(argv[arg], "-p") == 0) { partNum = atoi(argv[++arg]) - 1; partMax = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-L") == 0) { //minLen = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-E") == 0) { //minErate = atof(argv[++arg]); } else { fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); err++; } arg++; } if (err) { exit(1); } // Can't use dynamic threads, screws up the thread-local memory allocations. omp_set_dynamic(false); // Open gatekeeper store fprintf(stderr, "Opening '%s'\n", gkpName); gkpStore = gkStore::gkStore_open(gkpName); // Compute what to compute. if (partNum < partMax) { uint32 nf = gkpStore->gkStore_getNumReads(); iidMin = (partNum + 0) * nf / partMax + 1; iidMax = (partNum + 1) * nf / partMax; if (partNum + 1 == partMax) iidMax = nf; } if (iidMin == UINT32_MAX) iidMin = 1; if (iidMax == UINT32_MAX) iidMax = gkpStore->gkStore_getNumReads(); uint32 numIIDs = (iidMax - iidMin + 1); fprintf(stderr, " iidMin = %9u\n", iidMin); fprintf(stderr, " iidMax = %9u numReads = %9u\n", iidMax, gkpStore->gkStore_getNumReads()); fprintf(stderr, " partNum = %9u\n", partNum); fprintf(stderr, " partMax = %9u\n", partMax); //fprintf(stderr, "ovOverlap %lu\n", sizeof(ovOverlap)); //fprintf(stderr, "ESToverlap %lu\n", sizeof(ESToverlap)); // Load read metadata, clear ranges, read lengths, and deleted status. fprintf(stderr, "Initializing profiles\n"); uint64 readProfileSize = 0; readErrorEstimate *readProfile = new readErrorEstimate [numIIDs]; for (uint32 iid=0; iidgkStore_getRead(iid + iidMin)); if ((iid % 10000) == 0) fprintf(stderr, " %u reads\r", iid); } fprintf(stderr, " %u reads\n", numIIDs); fprintf(stderr, " %lu GB\n", readProfileSize >> 30); // Open overlap stores fprintf(stderr, "Opening '%s'\n", ovlStoreName); ovlStore = new ovStore(ovlStoreName, gkpStore); fprintf(stderr, "Finding number of overlaps\n"); ovlStore->setRange(iidMin, iidMax); uint64 numOvls = ovlStore->numOverlapsInRange(); uint64 *overlapIndex = new uint64 [numIIDs + 1]; uint32 bgn = 0; uint32 end = 0; uint32 *overlapLen = ovlStore->numOverlapsPerFrag(bgn, end); overlapIndex[0] = 0; for (uint32 iid=0; iid> 30); fprintf(stderr, " overlaps %lu GB (previous size)\n", (sizeof(ovOverlap) * numOvls) >> 30); fprintf(stderr, " overlaps %lu GB\n", (sizeof(ESToverlap) * numOvls) >> 30); ESToverlap *overlaps = NULL; memoryMappedFile *overlapsMMF = NULL; if (AS_UTL_fileExists(ovlCacheName, FALSE, FALSE)) { fprintf(stderr, " cache '%s' detected, load averted\n", ovlCacheName); overlapsMMF = new memoryMappedFile(ovlCacheName); overlaps = (ESToverlap *)overlapsMMF->get(0); } else { FILE *ESTcache = NULL; uint32 overlapblock = 100000000; ovOverlap *overlapsload = ovOverlap::allocateOverlaps(gkpStore, overlapblock); overlaps = new ESToverlap [numOvls]; if (ovlCacheName) { errno = 0; ESTcache = fopen(ovlCacheName, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", ovlCacheName, strerror(errno)), exit(1); } for (uint64 no=0; noreadOverlaps(overlapsload, overlapblock, false); for (uint32 xx=0; xx #include using namespace std; int main(int argc, char **argv) { char *gkpName = 0L; char *ovlName = 0L; char *tigName = 0L; uint32 tigVers = 0; uint32 errorRate = AS_OVS_encodeEvalue(0.015); char *outputPrefix = NULL; argc = AS_configure(argc, argv); uint32 iidMin = 0; uint32 iidMax = UINT32_MAX; uint32 numReadsPer = 0; uint32 numPartitions = 128; bool trimToAlign = true; int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { tigName = argv[++arg]; tigVers = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-o") == 0) { outputPrefix = argv[++arg]; } else if (strcmp(argv[arg], "-b") == 0) { iidMin = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-e") == 0) { iidMax = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-n") == 0) { numReadsPer = atoi(argv[++arg]); numPartitions = 0; } else if (strcmp(argv[arg], "-p") == 0) { numReadsPer = 0; numPartitions = atoi(argv[++arg]); } else { fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); err++; } arg++; } if (err) { exit(1); } if (gkpName == NULL || tigName == NULL) { exit(1); } // Open gkpStore. Pretty much the first thing we always do. gkStore *gkpStore = gkStore::gkStore_open(gkpName); // Open tigStore, check ranges. tgStore *tigStore = new tgStore(tigName, tigVers); uint32 nTigs = tigStore->numTigs(); if (nTigs <= iidMax) iidMax = nTigs - 1; // Count how many reads are referenced in tigs from iidMin to iidMax. These unitigs // are special in that the can contain duplicate reads, so there will be many more // reads referenced than the number of reads in gkpStore. uint64 nReadsInTigs = 0; for (uint32 ti=iidMin; ti<=iidMax; ti++) nReadsInTigs += tigStore->getNumChildren(ti); // Decide how many partitions there should be. Rather easy if the value is supplied, // but if not, compute it from the number of reads per partition. if (numReadsPer > 0) numPartitions = nReadsInTigs / numReadsPer + 1; fprintf(stderr, "Will partition "F_U64" total child reads into "F_U32" partitions.\n", nReadsInTigs, numPartitions); // Decide on a partitioning, based on total reads per tig. uint32 *tigToPart = new uint32 [nTigs]; uint32 *nReadsPerPart = new uint32 [numPartitions + 1]; memset(tigToPart, 0, sizeof(uint32) * (nTigs)); memset(nReadsPerPart, 0, sizeof(uint32) * (numPartitions + 1)); // Grab the number of reads per tig, again, but let us sort it to do a simple // greedy partitioning. vector > readsPerTig; for (uint32 ti=iidMin; ti<=iidMax; ti++) readsPerTig.push_back(pair(tigStore->getNumChildren(ti), ti)); sort(readsPerTig.rend(), readsPerTig.rbegin()); // Put the next unitig in the most empty partition. Definitely better algorithms exist... for (uint32 ii=0; iiloadTig(ti); if (tig == NULL) continue; if (tig->numberOfChildren() == 0) continue; uint32 pp = tigToPart[ti]; assert(pp > 0); if (partFile[pp] == NULL) { char name[FILENAME_MAX]; sprintf(name, "%s%04d", outputPrefix, pp); // Sync'd with canu/CorrectReads.pm errno = 0; partFile[pp] = fopen(name, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", name, strerror(errno)), exit(1); } outputFalcon(gkpStore, tig, trimToAlign, partFile[pp], readData); } delete readData; for (uint32 pp=0; pp<=numPartitions; pp++) { if (partFile[pp] == NULL) continue; fprintf(partFile[pp], "- -\n"); fclose(partFile[pp]); } delete tigStore; delete [] partFile; delete [] tigToPart; delete [] nReadsPerPart; gkpStore->gkStore_close(); return(0); } canu-1.3/src/falcon_sense/createFalconSenseInputs.mk000066400000000000000000000010101272335422300226440ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := createFalconSenseInputs SOURCES := createFalconSenseInputs.C outputFalcon.C SRC_INCDIRS := .. ../AS_UTL ../stores TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/falcon_sense/falcon_sense.C000066400000000000000000000066621272335422300203110ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2016-FEB-24 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "splitToWords.H" #include "AS_UTL_fasta.H" #include "falcon.H" #include #include #include using namespace std; int main (int argc, char **argv) { uint32 threads = 0; uint32 min_cov = 4; uint32 min_len = 500; uint32 min_ovl_len = 500; double min_idy = 0.5; uint32 K = 8; uint32 max_read_len = AS_MAX_READLEN; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "--n_core") == 0) { threads = atoi(argv[++arg]); } else if (strcmp(argv[arg], "--min_cov") == 0) { min_cov = atoi(argv[++arg]); } else if (strcmp(argv[arg], "--min_idt") == 0) { min_idy = atof(argv[++arg]); } else if (strcmp(argv[arg], "--min_len") == 0) { min_len = atoi(argv[++arg]); } else if (strcmp(argv[arg], "--min_ovl_len") == 0) { min_ovl_len = atoi(argv[++arg]); } else if (strcmp(argv[arg], "--max_read_len") == 0) { max_read_len = atoi(argv[++arg]); if (max_read_len <= 0 || max_read_len > AS_MAX_READLEN) { max_read_len = AS_MAX_READLEN; } } else { fprintf(stderr, "%s: Unknown option '%s'\n", argv[0], argv[arg]); err++; } arg++; } if (err) { fprintf(stderr, "Invalid usage"); exit(1); } if (threads > 0) { omp_set_num_threads(threads); } else { omp_set_num_threads(omp_get_max_threads()); } // read in a loop and get consensus of each read vector seqs; string seed; char *A = new char[AS_MAX_READLEN * 2]; while (true) { fgets(A, AS_MAX_READLEN * 2, stdin); splitToWords W(A); if (W[0][0] == '+') { uint32 splitSeqID = 0; FConsensus::consensus_data *consensus_data_ptr = FConsensus::generate_consensus( seqs, min_cov, K, min_idy, min_ovl_len, max_read_len ); char * split = strtok(consensus_data_ptr->sequence, "acgt"); while (split != NULL) { if (strlen(split) > min_len) { AS_UTL_writeFastA(stdout, split, strlen(split), 60, ">%s_%d\n", seed.c_str(), splitSeqID); splitSeqID++; } split = strtok(NULL, "acgt"); } FConsensus::free_consensus_data( consensus_data_ptr ); seqs.clear(); seed.clear(); } else if (W[0][0] == '-') { break; } else { if (seed.length() == 0) { seed = W[0]; seqs.push_back(string(W[1])); } else if (strlen(W[1]) > min_ovl_len) { seqs.push_back(string(W[1])); } } } delete[] A; } canu-1.3/src/falcon_sense/falcon_sense.mk000066400000000000000000000010041272335422300205170ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := falcon_sense SOURCES := falcon_sense.C SRC_INCDIRS := .. ../AS_UTL ../stores ../utgcns/libNDFalcon libfalcon TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/falcon_sense/libfalcon/000077500000000000000000000000001272335422300174655ustar00rootroot00000000000000canu-1.3/src/falcon_sense/libfalcon/falcon.C000066400000000000000000000570251272335422300210440ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2016-FEB-24 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ /* * ===================================================================================== * * Filename: fastcon.c * * Description: * * Version: 0.1 * Created: 07/20/2013 17:00:00 * Revision: none * Compiler: gcc * * Author: Jason Chin, * Company: * * ===================================================================================== #################################################################################$$ # Copyright (c) 2011-2014, Pacific Biosciences of California, Inc. # # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted (subject to the limitations in the # disclaimer below) provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # # * Neither the name of Pacific Biosciences nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE # GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC # BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED # WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. #################################################################################$$ */ #include "falcon.H" #include #include #include #include #include #include namespace FConsensus { typedef struct { seq_coor_t t_pos; uint8 delta; char q_base; seq_coor_t p_t_pos; // the tag position of the previous base uint8 p_delta; // the tag delta of the previous base char p_q_base; // the previous base uint32 q_id; } align_tag_t; typedef struct { seq_coor_t len; align_tag_t * align_tags; } align_tags_t; typedef struct { uint16 size; uint16 n_link; seq_coor_t * p_t_pos; // the tag position of the previous base uint8 * p_delta; // the tag delta of the previous base char * p_q_base; // the previous base uint16 * link_count; uint16 count; seq_coor_t best_p_t_pos; uint8 best_p_delta; uint8 best_p_q_base; // encoded base double score; } align_tag_col_t; typedef struct { align_tag_col_t * base; } msa_base_group_t; typedef struct { uint8 size; uint8 max_delta; msa_base_group_t * delta; } msa_delta_group_t; typedef msa_delta_group_t * msa_pos_t; align_tags_t * get_align_tags( char * aln_q_seq, char * aln_t_seq, seq_coor_t aln_seq_len, aln_range * range, uint32 q_id, seq_coor_t t_offset) { char p_q_base; align_tags_t * tags; seq_coor_t i, j, jj, k, p_j, p_jj; tags = (align_tags_t *)calloc( 1, sizeof(align_tags_t) ); tags->len = aln_seq_len; tags->align_tags = (align_tag_t *)calloc( aln_seq_len + 1, sizeof(align_tag_t) ); i = range->s1 - 1; j = range->s2 - 1; jj = 0; p_j = -1; p_jj = 0; p_q_base = '.'; for (k = 0; k < aln_seq_len; k++) { if (aln_q_seq[k] != '-') { i ++; jj ++; } if (aln_t_seq[k] != '-') { j ++; jj = 0; } //printf("t %d %d %d %c %c\n", q_id, j, jj, aln_t_seq[k], aln_q_seq[k]); if ( j + t_offset >= 0 && jj < uint8MAX && p_jj < uint8MAX) { (tags->align_tags[k]).t_pos = j + t_offset; (tags->align_tags[k]).delta = jj; (tags->align_tags[k]).p_t_pos = p_j + t_offset; (tags->align_tags[k]).p_delta = p_jj; (tags->align_tags[k]).p_q_base = p_q_base; (tags->align_tags[k]).q_base = aln_q_seq[k]; (tags->align_tags[k]).q_id = q_id; p_j = j; p_jj = jj; p_q_base = aln_q_seq[k]; } } // sentinal at the end //k = aln_seq_len; tags->len = k; (tags->align_tags[k]).t_pos = uint32MAX; (tags->align_tags[k]).delta = uint8MAX; (tags->align_tags[k]).q_base = '.'; (tags->align_tags[k]).q_id = uint32MAX; return tags; } void free_align_tags( align_tags_t * tags) { free( tags->align_tags ); free( tags ); } void allocate_aln_col( align_tag_col_t * col) { col->p_t_pos = ( seq_coor_t * ) calloc(col->size, sizeof( seq_coor_t )); col->p_delta = ( uint8 * ) calloc(col->size, sizeof( uint8 )); col->p_q_base = ( char * )calloc(col->size, sizeof( char )); col->link_count = ( uint16 * ) calloc(col->size, sizeof( uint16 )); } void realloc_aln_col( align_tag_col_t * col ) { col->p_t_pos = (seq_coor_t *) realloc( col->p_t_pos, (col->size) * sizeof( seq_coor_t )); col->p_delta = ( uint8 *) realloc( col->p_delta, (col->size) * sizeof( uint8 )); col->p_q_base = (char *) realloc( col->p_q_base, (col->size) * sizeof( char )); col->link_count = ( uint16 *) realloc( col->link_count, (col->size) * sizeof( uint16 )); } void free_aln_col( align_tag_col_t * col) { free(col->p_t_pos); free(col->p_delta); free(col->p_q_base); free(col->link_count); } void allocate_delta_group( msa_delta_group_t * g) { int i,j; g->max_delta = 0; g->delta = (msa_base_group_t *) calloc( g->size, sizeof(msa_base_group_t)); for (i = 0; i< g->size; i++) { g->delta[i].base = ( align_tag_col_t * ) calloc( 5, sizeof(align_tag_col_t ) ); for (j = 0; j < 5; j++ ) { g->delta[i].base[j].size = 8; allocate_aln_col(&(g->delta[i].base[j])); } } } void realloc_delta_group( msa_delta_group_t * g, uint16 new_size ) { int i, j, bs, es; bs = g->size; es = new_size; g->delta = (msa_base_group_t *) realloc(g->delta, new_size * sizeof(msa_base_group_t)); for (i=bs; i < es; i++) { g->delta[i].base = ( align_tag_col_t *) calloc( 5, sizeof(align_tag_col_t ) ); for (j = 0; j < 5; j++ ) { g->delta[i].base[j].size = 8; allocate_aln_col(&(g->delta[i].base[j])); } } g->size = new_size; } void free_delta_group( msa_delta_group_t * g) { //manything to do here int i, j; for (i = 0; i < g->size; i++) { for (j = 0; j < 5; j++) { free_aln_col( &(g->delta[i].base[j]) ); } free(g->delta[i].base); } free(g->delta); } void update_col( align_tag_col_t * col, seq_coor_t p_t_pos, uint8 p_delta, char p_q_base) { int updated = 0; int kk; col->count += 1; for (kk = 0; kk < col->n_link; kk++) { if ( p_t_pos == col->p_t_pos[kk] && p_delta == col->p_delta[kk] && p_q_base == col->p_q_base[kk] ) { col->link_count[kk] ++; updated = 1; break; } } if (updated == 0) { if (col->n_link + 1 > col->size) { if (col->size < (uint16MAX >> 1)-1) { col->size *= 2; } else { col->size += 256; } assert( col->size < uint16MAX-1 ); realloc_aln_col(col); } kk = col->n_link; col->p_t_pos[kk] = p_t_pos; col->p_delta[kk] = p_delta; col->p_q_base[kk] = p_q_base; col->link_count[kk] = 1; col->n_link++; } } msa_pos_t * get_msa_working_sapce(uint32 max_t_len) { msa_pos_t * msa_array; uint32 i; msa_array = (msa_pos_t *)calloc(max_t_len, sizeof(msa_pos_t)); for (i = 0; i < max_t_len; i++) { msa_array[i] = (msa_delta_group_t *)calloc(1, sizeof(msa_delta_group_t)); msa_array[i]->size = 8; allocate_delta_group(msa_array[i]); } return msa_array; } void clean_msa_working_space( msa_pos_t * msa_array, uint32 max_t_len) { uint32 i,j,k; align_tag_col_t * col; for (i = 0; i < max_t_len; i++) { for (j =0; j < msa_array[i]->max_delta + 1; j++) { for (k = 0; k < 5; k++ ) { col = msa_array[i]->delta[j].base + k; /* for (c =0; c < col->size; c++) { col->p_t_pos[c] = 0; col->p_delta[c] = 0; col->p_q_base[c] = 0; col->link_count[c] =0; } */ col->n_link = 0; col->count = 0; col->best_p_t_pos = 0; col->best_p_delta = 0; col->best_p_q_base = 0; col->score = 0; } } msa_array[i]->max_delta = 0; } } #define STATIC_ALLOCATE //#undef STATIC_ALLOCATE consensus_data * get_cns_from_align_tags( align_tags_t ** tag_seqs, uint32 n_tag_seqs, uint32 t_len, uint32 min_cov, uint32 max_len ) { seq_coor_t i,j; seq_coor_t t_pos = 0; seq_coor_t t_count = 0; uint32 * coverage; uint32 * local_nbase; consensus_data * consensus; align_tag_t * c_tag; static msa_pos_t * msa_array = NULL; // figure out true t_len and compact, we might have blank spaces for unaligned sequences for (i = 0; i < n_tag_seqs; i++) if (tag_seqs[i] != NULL) tag_seqs[t_count++] = tag_seqs[i]; // null out the remainder for (i = t_count; i < n_tag_seqs; i++) tag_seqs[i] = NULL; n_tag_seqs = t_count; if (n_tag_seqs == 0) { // allocate an empty consensus sequence consensus = (consensus_data *)calloc( 1, sizeof(consensus_data) ); consensus->sequence = (char *)calloc( 1, sizeof(char) ); consensus->eqv = (int32 *)calloc( 1, sizeof(int32) ); return consensus; } coverage = (uint32 *)calloc( t_len, sizeof(uint32) ); local_nbase = (uint32 *)calloc( t_len, sizeof(uint32) ); #ifndef STATIC_ALLOCATE msa_array = (msa_pos_t *)calloc(t_len, sizeof(msa_pos_t)); for (i = 0; i < t_len; i++) { msa_array[i] = (msa_delta_group_t *)calloc(1, sizeof(msa_delta_group_t)); msa_array[i]->size = 8; allocate_delta_group(msa_array[i]); } #endif #ifdef STATIC_ALLOCATE if ( msa_array == NULL) { msa_array = get_msa_working_sapce( max_len ); } assert(t_len < max_len); #endif // loop through every alignment //printf("XX %d\n", n_tag_seqs); for (i = 0; i < n_tag_seqs; i++) { // for each alignment position, insert the alignment tag to msa_array for (j = 0; j < tag_seqs[i]->len; j++) { c_tag = tag_seqs[i]->align_tags + j; uint32 delta; delta = c_tag->delta; if (delta == 0) { t_pos = c_tag->t_pos; coverage[ t_pos ] ++; } // Assume t_pos was set on earlier iteration. // (Otherwise, use its initial value, which might be an error. ~cd) if (delta > msa_array[t_pos]->max_delta) { msa_array[t_pos]->max_delta = delta; if (msa_array[t_pos]->max_delta + 4 > msa_array[t_pos]->size ) { realloc_delta_group(msa_array[t_pos], msa_array[t_pos]->max_delta + 8); } } uint32 base = -1; switch (c_tag->q_base) { case 'A': base = 0; break; case 'C': base = 1; break; case 'G': base = 2; break; case 'T': base = 3; break; case '-': base = 4; break; default : base = 4; break; } // Note: On bad input, base may be -1. update_col( &(msa_array[t_pos]->delta[delta].base[base]), c_tag->p_t_pos, c_tag->p_delta, c_tag->p_q_base); local_nbase[ t_pos ] ++; } } // propogate score throught the alignment links, setup backtracking information align_tag_col_t * g_best_aln_col = 0; uint32 g_best_ck = 0; seq_coor_t g_best_t_pos = 0; { int kk; int ck; // char base; int best_i; int best_j; int best_b; int best_ck = -1; double score; double best_score; double g_best_score; // char best_mark; align_tag_col_t * aln_col; g_best_score = -1; for (i = 0; i < t_len; i++) { //loop through every template base //printf("max delta: %d %d\n", i, msa_array[i]->max_delta); for (j = 0; j <= msa_array[i]->max_delta; j++) { // loop through every delta position for (kk = 0; kk < 5; kk++) { // loop through diff bases of the same delta posiiton /* switch (kk) { case 0: base = 'A'; break; case 1: base = 'C'; break; case 2: base = 'G'; break; case 3: base = 'T'; break; case 4: base = '-'; break; } */ aln_col = msa_array[i]->delta[j].base + kk; best_score = -1; best_i = -1; best_j = -1; best_b = -1; for (ck = 0; ck < aln_col->n_link; ck++) { // loop through differnt link to previous column int pi; int pj; int pkk; pi = aln_col->p_t_pos[ck]; pj = aln_col->p_delta[ck]; switch (aln_col->p_q_base[ck]) { case 'A': pkk = 0; break; case 'C': pkk = 1; break; case 'G': pkk = 2; break; case 'T': pkk = 3; break; case '-': pkk = 4; break; default : pkk = 4; break; } if (aln_col->p_t_pos[ck] == -1) { score = (double) aln_col->link_count[ck] - (double) coverage[i] * 0.5; } else if (pj > msa_array[pi]->max_delta) { score = (double) aln_col->link_count[ck] - (double) coverage[i] * 0.5; } else { score = msa_array[pi]->delta[pj].base[pkk].score + (double) aln_col->link_count[ck] - (double) coverage[i] * 0.5; } // best_mark = ' '; if (score > best_score) { best_score = score; aln_col->best_p_t_pos = best_i = pi; aln_col->best_p_delta = best_j = pj; aln_col->best_p_q_base = best_b = pkk; best_ck = ck; // best_mark = '*'; } /* printf("X %d %d %d %c %d %d %d %c %d %lf %c\n", coverage[i], i, j, base, aln_col->count, aln_col->p_t_pos[ck], aln_col->p_delta[ck], aln_col->p_q_base[ck], aln_col->link_count[ck], score, best_mark); */ } aln_col->score = best_score; if (best_score > g_best_score) { g_best_score = best_score; g_best_aln_col = aln_col; g_best_ck = best_ck; g_best_t_pos = i; //printf("GB %d %d %d %d\n", i, j, ck, g_best_aln_col); } } } } assert(g_best_score != -1); } // reconstruct the sequences uint32 index; char bb = '$'; int ck; char * cns_str; int * eqv; double score0; consensus = (consensus_data *)calloc( 1, sizeof(consensus_data) ); consensus->sequence = (char *)calloc( t_len * 2 + 1, sizeof(char) ); consensus->eqv = (int32 *)calloc( t_len * 2 + 1, sizeof(int32) ); cns_str = consensus->sequence; eqv = consensus->eqv; index = 0; ck = g_best_ck; i = g_best_t_pos; while (1) { if (coverage[i] > min_cov) { switch (ck) { case 0: bb = 'A'; break; case 1: bb = 'C'; break; case 2: bb = 'G'; break; case 3: bb = 'T'; break; case 4: bb = '-'; break; } } else { switch (ck) { case 0: bb = 'a'; break; case 1: bb = 'c'; break; case 2: bb = 'g'; break; case 3: bb = 't'; break; case 4: bb = '-'; break; } } // Note: On bad input, bb will keep previous value, possibly '$'. score0 = g_best_aln_col->score; i = g_best_aln_col->best_p_t_pos; if (i == -1 || index >= t_len * 2) break; j = g_best_aln_col->best_p_delta; ck = g_best_aln_col->best_p_q_base; g_best_aln_col = msa_array[i]->delta[j].base + ck; if (bb != '-') { cns_str[index] = bb; eqv[index] = (int) score0 - (int) g_best_aln_col->score; //printf("C %d %d %c %lf %d %d\n", i, index, bb, g_best_aln_col->score, coverage[i], eqv[index] ); index ++; } } // reverse the sequence for (i = 0; i < index/2; i++) { cns_str[i] = cns_str[i] ^ cns_str[index-i-1]; cns_str[index-i-1] = cns_str[i] ^ cns_str[index-i-1]; cns_str[i] = cns_str[i] ^ cns_str[index-i-1]; eqv[i] = eqv[i] ^ eqv[index-i-1]; eqv[index-i-1] = eqv[i] ^ eqv[index-i-1]; eqv[i] = eqv[i] ^ eqv[index-i-1]; } cns_str[index] = 0; //printf("%s\n", cns_str); #ifndef STATIC_ALLOCATE for (i = 0; i < t_len; i++) { free_delta_group(msa_array[i]); free(msa_array[i]); } free(msa_array); #endif #ifdef STATIC_ALLOCATE clean_msa_working_space(msa_array, t_len+1); #endif free(coverage); free(local_nbase); return consensus; } consensus_data * generate_consensus( vector input_seq, uint32 min_cov, uint32 K, double min_idt, uint32 min_len, uint32 max_len) { uint32 seq_count; kmer_lookup * lk_ptr; seq_array sa_ptr; seq_addr_array sda_ptr; align_tags_t ** tags_list; consensus_data * consensus; double max_diff; max_diff = 1.0 - min_idt; seq_count = input_seq.size(); fflush(stdout); tags_list = (align_tags_t **)calloc( seq_count, sizeof(align_tags_t*) ); lk_ptr = allocate_kmer_lookup( 1 << (K * 2) ); sa_ptr = allocate_seq( (seq_coor_t) input_seq[0].length() ); sda_ptr = allocate_seq_addr( (seq_coor_t) input_seq[0].length() ); add_sequence( 0, K, input_seq[0].c_str(), input_seq[0].length(), sda_ptr, sa_ptr, lk_ptr); #pragma omp parallel for schedule(dynamic) for (uint32 j=0; j < seq_count; j++) { #define MAX_UNMASKED_LENGTH 500000 #define MAX_KMER_REPEAT 1000 if (input_seq[j].length() > MAX_UNMASKED_LENGTH) { mask_k_mer(1 << (K*2), lk_ptr, MAX_KMER_REPEAT); } kmer_match *kmer_match_ptr = find_kmer_pos_for_seq(input_seq[j].c_str(), input_seq[j].length(), K, sda_ptr, lk_ptr); #define INDEL_ALLOWENCE_0 6 aln_range *arange = find_best_aln_range(kmer_match_ptr, K, K * INDEL_ALLOWENCE_0, 5); // narrow band to avoid aligning through big indels //fprintf(stderr, "1:read %d %ld %ld %ld %ld\n", j, arange->s1, arange->e1, arange->s2, arange->e2); //arange = find_best_aln_range2(kmer_match_ptr, K, K * INDEL_ALLOWENCE_0, 5); // narrow band to avoid aligning through big indels //printf("2:%ld %ld %ld %ld\n\n", arange->s1, arange->e1, arange->s2, arange->e2); #define INDEL_ALLOWENCE_1 0.10 if (arange->e1 - arange->s1 < 100 || arange->e2 - arange->s2 < 100 || abs( (arange->e1 - arange->s1 ) - (arange->e2 - arange->s2) ) > (int) (0.5 * INDEL_ALLOWENCE_1 * (arange->e1 - arange->s1 + arange->e2 - arange->s2))) { free_kmer_match( kmer_match_ptr); free_aln_range(arange); continue; } #define INDEL_ALLOWENCE_2 150 NDalignment::NDalignResult aln; align(input_seq[j].c_str()+arange->s1, arange->e1 - arange->s1 , input_seq[0].c_str()+arange->s2, arange->e2 - arange->s2 , INDEL_ALLOWENCE_2, 1, aln); if (aln._size > min_len && ((double) aln._dist / (double) aln._size) < max_diff) { tags_list[j] = get_align_tags( aln._qry_aln_str, aln._tgt_aln_str, aln._size, arange, j, 0); //fprintf(stderr, "Aligned seq %d to positions %d - %d and %d - %d with %d diffs size %d\n", j, aln._qry_bgn, aln._qry_end, aln._tgt_bgn, aln._tgt_end, aln._dist, aln._size); } free_aln_range(arange); free_kmer_match( kmer_match_ptr); } consensus = get_cns_from_align_tags( tags_list, seq_count, input_seq[0].length(), min_cov, max_len); free_seq_addr_array(sda_ptr); free_seq_array(sa_ptr); free_kmer_lookup(lk_ptr); for (int j=0; j < seq_count; j++) if (tags_list[j] != NULL) free_align_tags(tags_list[j]); free(tags_list); return consensus; } void free_consensus_data( consensus_data * consensus ){ free(consensus->sequence); free(consensus->eqv); free(consensus); } } canu-1.3/src/falcon_sense/libfalcon/falcon.H000066400000000000000000000124441272335422300210450ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2016-FEB-24 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ /* * ===================================================================================== * * Filename: common.h * * Description: Common delclaration for the code base * * Version: 0.1 * Created: 07/16/2013 07:46:23 AM * Revision: none * Compiler: gcc * * Author: Jason Chin, * Company: * * ===================================================================================== #################################################################################$$ # Copyright (c) 2011-2014, Pacific Biosciences of California, Inc. # # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted (subject to the limitations in the # disclaimer below) provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # # * Neither the name of Pacific Biosciences nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE # GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC # BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED # WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. #################################################################################$$ */ #include "AS_global.H" #include "dw.H" #include #include using namespace std; using namespace NDalignment; namespace FConsensus { typedef struct { seq_coor_t start; seq_coor_t last; seq_coor_t count; } kmer_lookup; typedef unsigned char base; typedef base * seq_array; typedef seq_coor_t seq_addr; typedef seq_addr * seq_addr_array; typedef struct { seq_coor_t count; seq_coor_t * query_pos; seq_coor_t * target_pos; } kmer_match; typedef struct { seq_coor_t s1; seq_coor_t e1; seq_coor_t s2; seq_coor_t e2; long int score; } aln_range; typedef struct { char * sequence; int32 * eqv; } consensus_data; kmer_lookup * allocate_kmer_lookup (seq_coor_t); void init_kmer_lookup ( kmer_lookup *, seq_coor_t ); void free_kmer_lookup(kmer_lookup *); seq_array allocate_seq(seq_coor_t); void init_seq_array( seq_array, seq_coor_t); void free_seq_array(seq_array); seq_addr_array allocate_seq_addr(seq_coor_t size); void free_seq_addr_array(seq_addr_array); aln_range * find_best_aln_range(kmer_match *, seq_coor_t, seq_coor_t, seq_coor_t); void free_aln_range( aln_range *); kmer_match * find_kmer_pos_for_seq( const char *, seq_coor_t, uint32 K, seq_addr_array, kmer_lookup * ); void free_kmer_match( kmer_match * ptr); void free_kmer_lookup(kmer_lookup * ); void add_sequence ( seq_coor_t, uint32, const char *, seq_coor_t, seq_addr_array, seq_array, kmer_lookup *); void mask_k_mer(seq_coor_t, kmer_lookup *, seq_coor_t); consensus_data * generate_consensus( vector input_seq, uint32 min_cov, uint32 K, double min_idt, uint32 min_len, uint32 max_len); void free_consensus_data(consensus_data *); } canu-1.3/src/falcon_sense/libfalcon/kmer_lookup.C000066400000000000000000000456771272335422300221430ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2016-FEB-24 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ /* * ===================================================================================== * * Filename: kmer_count.c * * Description: * * Version: 0.1 * Created: 07/20/2013 17:00:00 * Revision: none * Compiler: gcc * * Author: Jason Chin, * Company: * * ===================================================================================== #################################################################################$$ # Copyright (c) 2011-2014, Pacific Biosciences of California, Inc. # # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted (subject to the limitations in the # disclaimer below) provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # # * Neither the name of Pacific Biosciences nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE # GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC # BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED # WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. #################################################################################$$ */ #include #include #include #include "falcon.H" namespace FConsensus { const uint32 KMERMATCHINC = 10000; int compare_seq_coor(const void * a, const void * b) { const seq_coor_t * arg1 = (const seq_coor_t *)a; const seq_coor_t * arg2 = (const seq_coor_t *)b; return (* arg1) - (* arg2); } kmer_lookup * allocate_kmer_lookup ( seq_coor_t size ) { kmer_lookup * kl; kl = (kmer_lookup *) malloc( size * sizeof(kmer_lookup) ); init_kmer_lookup( kl, size); return kl; } void init_kmer_lookup ( kmer_lookup * kl, seq_coor_t size ) { seq_coor_t i; for (i=0; i threshold) { kl[i].start = INT_MAX; kl[i].last = INT_MAX; //kl[i].count = 0; } } } kmer_match * find_kmer_pos_for_seq( const char * seq, seq_coor_t seq_len, uint32 K, seq_addr_array sda, kmer_lookup * lk) { seq_coor_t i; seq_coor_t kmer_bv; seq_coor_t kmer_mask; seq_coor_t kmer_pos; seq_coor_t next_kmer_pos; uint32 half_K; seq_coor_t kmer_match_rtn_allocation_size = KMERMATCHINC; kmer_match * kmer_match_rtn; base * sa; kmer_match_rtn = (kmer_match *) malloc( sizeof(kmer_match) ); kmer_match_rtn->count = 0; kmer_match_rtn->query_pos = (seq_coor_t *) calloc( kmer_match_rtn_allocation_size, sizeof( seq_coor_t ) ); kmer_match_rtn->target_pos = (seq_coor_t *) calloc( kmer_match_rtn_allocation_size, sizeof( seq_coor_t ) ); sa = (base *)calloc( seq_len, sizeof(base) ); kmer_mask = 0; for (i = 0; i < K; i++) { kmer_mask <<= 2; kmer_mask |= 0x00000003; } for (i = 0; i < seq_len; i++) { switch ( seq[i] ) { case 'A': sa[ i ] = 0; break; case 'C': sa[ i ] = 1; break; case 'G': sa[ i ] = 2; break; case 'T': sa[ i ] = 3; break; default: sa[ i ] = 0; } } kmer_bv = get_kmer_bitvector(sa, K); half_K = K >> 1; for (i = 0; i < seq_len - K; i += half_K) { kmer_bv = get_kmer_bitvector(sa + i, K); if (lk[kmer_bv].start == INT_MAX) { //for high count k-mers continue; } kmer_pos = lk[ kmer_bv ].start; next_kmer_pos = sda[ kmer_pos ]; kmer_match_rtn->query_pos[ kmer_match_rtn->count ] = i; kmer_match_rtn->target_pos[ kmer_match_rtn->count ] = kmer_pos; kmer_match_rtn->count += 1; if (kmer_match_rtn->count > kmer_match_rtn_allocation_size - 1000) { kmer_match_rtn_allocation_size += KMERMATCHINC; kmer_match_rtn->query_pos = (seq_coor_t *) realloc( kmer_match_rtn->query_pos, kmer_match_rtn_allocation_size * sizeof(seq_coor_t) ); kmer_match_rtn->target_pos = (seq_coor_t *) realloc( kmer_match_rtn->target_pos, kmer_match_rtn_allocation_size * sizeof(seq_coor_t) ); } while ( next_kmer_pos > kmer_pos ){ kmer_pos = next_kmer_pos; next_kmer_pos = sda[ kmer_pos ]; kmer_match_rtn->query_pos[ kmer_match_rtn->count ] = i; kmer_match_rtn->target_pos[ kmer_match_rtn->count ] = kmer_pos; kmer_match_rtn->count += 1; if (kmer_match_rtn->count > kmer_match_rtn_allocation_size - 1000) { kmer_match_rtn_allocation_size += KMERMATCHINC; kmer_match_rtn->query_pos = (seq_coor_t *) realloc( kmer_match_rtn->query_pos, kmer_match_rtn_allocation_size * sizeof(seq_coor_t) ); kmer_match_rtn->target_pos = (seq_coor_t *) realloc( kmer_match_rtn->target_pos, kmer_match_rtn_allocation_size * sizeof(seq_coor_t) ); } } } free(sa); return kmer_match_rtn; } void free_kmer_match( kmer_match * ptr) { free(ptr->query_pos); free(ptr->target_pos); free(ptr); } aln_range* find_best_aln_range(kmer_match * km_ptr, seq_coor_t K, seq_coor_t bin_size, seq_coor_t count_th) { seq_coor_t i; seq_coor_t j; seq_coor_t q_min, q_max, t_min, t_max; seq_coor_t * d_count; seq_coor_t * q_coor; seq_coor_t * t_coor; aln_range * arange; long int d, d_min, d_max; long int cur_score; long int max_score; long int max_k_mer_count; long int max_k_mer_bin; seq_coor_t cur_start; arange = (aln_range *)calloc(1 , sizeof(aln_range)); q_min = INT_MAX; q_max = 0; t_min = INT_MAX; t_max = 0; d_min = INT_MAX; d_max = LONG_MIN; for (i = 0; i < km_ptr->count; i++ ) { if ( km_ptr -> query_pos[i] < q_min) { q_min = km_ptr->query_pos[i]; } if ( km_ptr -> query_pos[i] > q_max) { q_max = km_ptr->query_pos[i]; } if ( km_ptr -> target_pos[i] < t_min) { t_min = km_ptr->target_pos[i]; } if ( km_ptr -> target_pos[i] > t_max) { t_max = km_ptr->target_pos[i]; } d = (long int) km_ptr->query_pos[i] - (long int) km_ptr->target_pos[i]; if ( d < d_min ) { d_min = d; } if ( d > d_max ) { d_max = d; } } //printf("%lu %ld %ld\n" , km_ptr->count, d_min, d_max); if (km_ptr->count == 0) d_max = d_min = 0; d_count = (seq_coor_t *)calloc( (d_max - d_min)/bin_size + 1, sizeof(seq_coor_t) ); q_coor = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) ); t_coor = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) ); for (i = 0; i < km_ptr->count; i++ ) { d = (long int) (km_ptr->query_pos[i]) - (long int) (km_ptr->target_pos[i]); d_count[ (d - d_min)/ (long int) bin_size ] += 1; q_coor[i] = INT_MAX; t_coor[i] = INT_MAX; } j = 0; max_k_mer_count = 0; max_k_mer_bin = INT_MAX; for (i = 0; i < km_ptr->count; i++ ) { d = (long int) (km_ptr->query_pos[i]) - (long int) (km_ptr->target_pos[i]); if ( d_count[ (d - d_min)/ (long int) bin_size ] > max_k_mer_count) { max_k_mer_count = d_count[ (d - d_min)/ (long int) bin_size ]; max_k_mer_bin = (d - d_min)/ (long int) bin_size; } } //printf("k_mer: %lu %lu\n" , max_k_mer_count, max_k_mer_bin); if ( max_k_mer_bin != INT_MAX && max_k_mer_count > count_th ) { for (i = 0; i < km_ptr->count; i++ ) { d = (long int) (km_ptr->query_pos[i]) - (long int) (km_ptr->target_pos[i]); if ( abs( ( (d - d_min)/ (long int) bin_size ) - max_k_mer_bin ) > 5 ) { continue; } if (d_count[ (d - d_min)/ (long int) bin_size ] > count_th) { q_coor[j] = km_ptr->query_pos[i]; t_coor[j] = km_ptr->target_pos[i]; //printf("d_count: %lu %lu\n" ,i, d_count[(d - d_min)/ (long int) bin_size]); //printf("coor: %lu %lu\n" , q_coor[j], t_coor[j]); j ++; } } } if (j > 1) { arange->s1 = q_coor[0]; arange->e1 = q_coor[0]; arange->s2 = t_coor[0]; arange->e2 = t_coor[0]; arange->score = 0; max_score = 0; cur_score = 0; cur_start = 0; for (i = 1; i < j; i++) { cur_score += 32 - (q_coor[i] - q_coor[i-1]); //printf("deltaD, %lu %ld\n", q_coor[i] - q_coor[i-1], cur_score); if (cur_score < 0) { cur_score = 0; cur_start = i; } else if (cur_score > max_score) { arange->s1 = q_coor[cur_start]; arange->s2 = t_coor[cur_start]; arange->e1 = q_coor[i]; arange->e2 = t_coor[i]; max_score = cur_score; arange->score = max_score; //printf("%lu %lu %lu %lu\n", arange.s1, arange.e1, arange.s2, arange.e2); } } } else { arange->s1 = 0; arange->e1 = 0; arange->s2 = 0; arange->e2 = 0; arange->score = 0; } // printf("free\n"); free(d_count); free(q_coor); free(t_coor); return arange; } aln_range* find_best_aln_range2(kmer_match * km_ptr, seq_coor_t K, seq_coor_t bin_width, seq_coor_t count_th) { seq_coor_t * d_coor; seq_coor_t * hit_score; seq_coor_t * hit_count; seq_coor_t * last_hit; seq_coor_t max_q, max_t; seq_coor_t s, e, max_s, max_e, max_span, d_s, d_e, delta, d_len; seq_coor_t px, py, cx, cy; seq_coor_t max_hit_idx; seq_coor_t max_hit_score, max_hit_count; seq_coor_t i, j; seq_coor_t candidate_idx, max_d, d; aln_range * arange; arange = (aln_range *)calloc(1 , sizeof(aln_range)); d_coor = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) ); max_q = -1; max_t = -1; for (i = 0; i < km_ptr->count; i++ ) { d_coor[i] = km_ptr->query_pos[i] - km_ptr->target_pos[i]; max_q = max_q > km_ptr->query_pos[i] ? max_q : km_ptr->query_pos[i]; max_t = max_t > km_ptr->target_pos[i] ? max_q : km_ptr->target_pos[i]; } qsort(d_coor, km_ptr->count, sizeof(seq_coor_t), compare_seq_coor); s = 0; e = 0; max_s = -1; max_e = -1; max_span = -1; delta = (long int) ( 0.05 * ( max_q + max_t ) ); d_len = km_ptr->count; d_s = -1; d_e = -1; while (1) { d_s = d_coor[s]; d_e = d_coor[e]; while (d_e < d_s + delta && e < d_len-1) { e += 1; d_e = d_coor[e]; } if ( max_span == -1 || e - s > max_span ) { max_span = e - s; max_s = s; max_e = e; } s += 1; if (s == d_len || e == d_len) { break; } } if (max_s == -1 || max_e == -1 || max_e - max_s < 32) { arange->s1 = 0; arange->e1 = 0; arange->s2 = 0; arange->e2 = 0; arange->score = 0; free(d_coor); return arange; } last_hit = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) ); hit_score = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) ); hit_count = (seq_coor_t *)calloc( km_ptr->count, sizeof(seq_coor_t) ); for (i = 0; i < km_ptr->count; i++ ) { last_hit[i] = -1; hit_score[i] = 0; hit_count[i] = 0; } max_hit_idx = -1; max_hit_score = 0; for (i = 0; i < km_ptr->count; i ++) { cx = km_ptr->query_pos[i]; cy = km_ptr->target_pos[i]; d = cx - cy; if ( d < d_coor[max_s] || d > d_coor[max_e] ) continue; j = i - 1; candidate_idx = -1; max_d = 65535; while (1) { if ( j < 0 ) break; px = km_ptr->query_pos[j]; py = km_ptr->target_pos[j]; d = px - py; if ( d < d_coor[max_s] || d > d_coor[max_e] ) { j--; continue; } if (cx - px > 320) break; //the number here controling how big alignment gap to be considered if (cy > py && cx - px + cy - py < max_d && cy - py <= 320 ) { max_d = cx - px + cy - py; candidate_idx = j; } j--; } if (candidate_idx != -1) { last_hit[i] = candidate_idx; hit_score[i] = hit_score[candidate_idx] + (64 - max_d); hit_count[i] = hit_count[candidate_idx] + 1; if (hit_score[i] < 0) { hit_score[i] = 0; hit_count[i] = 0; } } else { hit_score[i] = 0; hit_count[i] = 0; } if (hit_score[i] > max_hit_score) { max_hit_score = hit_score[i]; max_hit_count = hit_count[i]; max_hit_idx = i; } } if (max_hit_idx == -1) { arange->s1 = 0; arange->e1 = 0; arange->s2 = 0; arange->e2 = 0; arange->score = 0; free(d_coor); free(last_hit); free(hit_score); free(hit_count); return arange; } arange->score = max_hit_count + 1; arange->e1 = km_ptr->query_pos[max_hit_idx]; arange->e2 = km_ptr->target_pos[max_hit_idx]; i = max_hit_idx; while (last_hit[i] != -1) { i = last_hit[i]; } arange->s1 = km_ptr->query_pos[i]; arange->s2 = km_ptr->target_pos[i]; free(d_coor); free(last_hit); free(hit_score); free(hit_count); return arange; } void free_aln_range( aln_range * arange) { free(arange); } } canu-1.3/src/falcon_sense/outputFalcon.C000066400000000000000000000051041272335422300203230ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-APR-20 to 2015-MAY-20 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "outputFalcon.H" #include "AS_UTL_reverseComplement.H" // The falcon consensus format: // // name sequence // read sequence // read sequence // read sequence // read sequence // + + # generate consensus for the 'name' sequence using 'read' sequences // ... // - - # To end processing // void outputFalcon(gkStore *gkpStore, tgTig *tig, bool trimToAlign, FILE *F, gkReadData *readData) { gkpStore->gkStore_loadReadData(tig->tigID(), readData); fprintf(F, "read"F_U32" %s\n", tig->tigID(), readData->gkReadData_getSequence()); for (uint32 cc=0; ccnumberOfChildren(); cc++) { tgPosition *child = tig->getChild(cc); gkpStore->gkStore_loadReadData(child->ident(), readData); if (child->isReverse()) reverseComplementSequence(readData->gkReadData_getSequence(), readData->gkReadData_getRead()->gkRead_sequenceLength()); // For debugging/testing, skip one orientation of overlap. // //if (child->isReverse() == false) // continue; //if (child->isReverse() == true) // continue; // Trim the read to the aligned bit char *seq = readData->gkReadData_getSequence(); if (trimToAlign) { seq += child->_askip; seq[ readData->gkReadData_getRead()->gkRead_sequenceLength() - child->_askip - child->_bskip ] = 0; } fprintf(F, "data"F_U32" %s\n", tig->getChild(cc)->ident(), seq); } fprintf(F, "+ +\n"); } canu-1.3/src/falcon_sense/outputFalcon.H000066400000000000000000000023251272335422300203320ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2015-APR-20 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef OUTPUT_FALCON_H #define OUTPUT_FALCON_H #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" void outputFalcon(gkStore *gkpStore, tgTig *tig, bool trimToAlign, FILE *F, gkReadData *readData); #endif // OUTPUT_FALCON_H canu-1.3/src/fastq-utilities/000077500000000000000000000000001272335422300162245ustar00rootroot00000000000000canu-1.3/src/fastq-utilities/build-ordered-reads.sh000066400000000000000000000237771272335422300224150ustar00rootroot00000000000000#!/bin/sh # # Given a set of trimmed reads, map them to reference, filter out any that do not # map completely, and generate an output set of reads that is ordered and oriented. # if [ -d "/work/scripts" ] ; then bin=/work/wgspb/FreeBSD-amd64/bin scp=/work/pacbio-overlapper/scripts export PATH=${PATH}:/work/scripts fi if [ -d "/usr/local/projects/CELERA/bri/scripts" ] ; then bin=/usr/local/projects/CELERA/bri/wgspb/Linux-amd64/bin scp=/usr/local/projects/CELERA/bri/pacbio-assembler/scripts export PATH=${PATH}:/usr/local/projects/CELERA/bri/scripts fi if [ -z $bin ] ; then echo scripts not found. exit fi if [ -z $1 ] ; then echo usage: $0 work-directory reads.fastq reference.fasta exit fi wrk=$1 # Working directory name pfx="test" # Prefix of the files inp=$2 # Path to input FASTQ reads ref=$3 # Path to input FASTA reference if [ ! -d $wrk ] ; then mkdir $wrk fi if [ ! -e $inp ] ; then echo "Failed to find input FASTQ reads in '$inp'" exit fi if [ ! -e $ref ] ; then echo "" echo "WARNING: no reference, will not build ordered assembly." echo fi if [ ! -e "$wrk/build-ordered-reads.spec" ] ; then echo $wrk/build-ordered-reads.spec cat > $wrk/build-ordered-reads.spec \ < $wrk/BO/$pfx.frg echo gatekeeper -T -F -o $wrk/BO/$pfx.gkpStore $wrk/BO/$pfx.frg $bin/gatekeeper -T -F -o $wrk/BO/$pfx.gkpStore $wrk/BO/$pfx.frg fi if [ ! -e "$wrk/BO/$pfx.fastq" ] ; then echo gatekeeper -dumpfastq $wrk/BO/$pfx $wrk/BO/$pfx.gkpStore $bin/gatekeeper -dumpfastq $wrk/BO/$pfx $wrk/BO/$pfx.gkpStore awk '{ print $1 }' < $wrk/BO/$pfx.unmated.fastq > $wrk/BO/$pfx.fastq rm -f $wrk/BO/$pfx.unmated.fastq rm -f $wrk/BO/$pfx.paired.fastq rm -f $wrk/BO/$pfx.1.fastq rm -f $wrk/BO/$pfx.2.fastq echo replaceUIDwithName $wrk/BO/$pfx.gkpStore.fastqUIDmap $wrk/BO/$pfx.fastq $bin/replaceUIDwithName $wrk/BO/$pfx.gkpStore.fastqUIDmap $wrk/BO/$pfx.fastq fi # # OLD blasr options # # -maxLCPLength 15 # -nCandidates 25 # -maxScore -500 # if [ -e $ref ] ; then if [ ! -e "$wrk/BO/$pfx.blasr.badnm.sam" ] ; then echo \ blasr \ -noSplitSubreads \ -nproc 4 \ -minMatch 12 \ -bestn 10 \ -nCandidates 25 \ -minPctIdentity 65.0 \ -sam -clipping soft \ -out $wrk/BO/$pfx.blasr.badnm.sam \ $wrk/BO/$pfx.fastq \ $ref blasr \ -noSplitSubreads \ -nproc 4 \ -minMatch 12 \ -bestn 10 \ -minPctIdentity 65.0 \ -sam -clipping soft \ -out $wrk/BO/$pfx.blasr.badnm.sam \ $wrk/BO/$pfx.fastq \ $ref fi if [ ! -e "$wrk/BO/$pfx.blasr.sam" ] ; then echo a echo \ samtools calmd \ -S $wrk/BO/$pfx.blasr.badnm.sam \ $ref \ \> $wrk/BO/$pfx.blasr.sam samtools calmd \ -S $wrk/BO/$pfx.blasr.badnm.sam \ $ref \ > $wrk/BO/$pfx.blasr.sam fi if [ ! -e "$wrk/BO/$pfx.blasr.sam.coords" ] ; then echo bowtie2-to-nucmercoords.pl $ref \< $wrk/BO/$pfx.blasr.sam \> $wrk/BO/$pfx.blasr.sam.coords bowtie2-to-nucmercoords.pl $ref < $wrk/BO/$pfx.blasr.sam > $wrk/BO/$pfx.blasr.sam.coords fi if [ ! -e "$wrk/BO/$pfx.blasr.sam.coords.ova" ] ; then echo infer-olaps-from-coords perl $scp/infer-olaps-from-genomic-coords.pl \ $wrk/BO/$pfx.blasr.sam.coords \ $wrk/BO/$pfx.fastq \ $wrk/BO/$pfx.blasr.sam.coords \ $wrk/BO/$pfx.gkpStore.fastqUIDmap fi fi ######################################## # # Start an assembly using the blasr overlaps for the perfectly mapping reads. # # We need to rebuild the overlaps to get the IIDs correct for this assembly. # If we were to just use the BO/gkpStore and BO/ovlStore, all the reads that # don't map perfectly end up as singletons. # # For this to work, it is critical that the reads have their original names, NOT UIDs. # # NOTE! Reading coords and lengths from the BL work above, and using reads from the BO work here. # if [ -e $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq ] ; then if [ ! -d $wrk/BL ] ; then mkdir $wrk/BL fi if [ ! -d "$wrk/BL/$pfx.gkpStore" ] ; then $bin/fastqToCA \ -libraryname L \ -technology sanger \ -type sanger \ -reads $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq \ > $wrk/BL/$pfx.frg $bin/gatekeeper -T -F \ -o $wrk/BL/$pfx.gkpStore \ $wrk/BL/$pfx.frg fi if [ ! -e "$wrk/BL/$pfx.blasr.sam.coords.ova" ] ; then perl $scp/infer-olaps-from-genomic-coords.pl \ $wrk/BL/$pfx.blasr.sam.coords \ $wrk/BO/$pfx.fastq \ $wrk/BO/$pfx.blasr.sam.coords \ $wrk/BL/$pfx.gkpStore.fastqUIDmap fi if [ ! -d "$wrk/BL/$pfx.ovlStore" ] ; then $bin/convertOverlap -ovl \ < $wrk/BL/$pfx.blasr.sam.coords.ova \ > $wrk/BL/$pfx.blasr.sam.coords.ovb $bin/overlapStoreBuild \ -o $wrk/BL/$pfx.ovlStore \ -g $wrk/BL/$pfx.gkpStore \ -F 1 \ $wrk/BL/$pfx.blasr.sam.coords.ovb fi if [ ! -d "$wrk/BL/$pfx.tigStore" ] ; then perl $bin/runCA -p $pfx -d $wrk/BL -s $wrk/build-ordered-reads.spec \ useGrid=0 scriptOnGrid=0 \ stopAfter=unitigger \ $wrk/BL/$pfx.frg else if [ ! -e "$wrk/BL/$pfx.qc" ] ; then perl $bin/runCA -p $pfx -d $wrk/BL -s $wrk/build-ordered-reads.spec \ useGrid=0 scriptOnGrid=0 \ $wrk/BL/$pfx.frg fi fi fi ######################################## # # Assembly with all reads. # if [ ! -d $wrk/CA ] ; then mkdir $wrk/CA fi if [ ! -d "$wrk/CA/$pfx.frg" ] ; then $bin/fastqToCA \ -libraryname L \ -technology sanger \ -type sanger \ -reads $inp \ > $wrk/CA/$pfx.frg fi if [ ! -d "$wrk/CA/$pfx.tigStore" ] ; then perl $bin/runCA -p $pfx -d $wrk/CA -s $wrk/build-ordered-reads.spec \ useGrid=0 scriptOnGrid=0 \ stopAfter=unitigger \ $wrk/CA/$pfx.frg else if [ ! -e "$wrk/CA/$pfx.qc" ] ; then perl $bin/runCA -p $pfx -d $wrk/CA -s $wrk/build-ordered-reads.spec \ useGrid=0 scriptOnGrid=0 \ $wrk/CA/$pfx.frg fi fi ######################################## # # Assembly with all reads. # if [ ! -d $wrk/CAcorrected ] ; then mkdir $wrk/CAcorrected fi if [ ! -d "$wrk/CAcorrected/$pfx.frg" ] ; then $bin/fastqToCA \ -libraryname L \ -technology sanger \ -type sanger \ -reads $inp \ > $wrk/CAcorrected/$pfx.frg fi if [ ! -d "$wrk/CAcorrected/$pfx.tigStore" ] ; then perl $bin/runCA -p $pfx -d $wrk/CAcorrected -s $wrk/build-ordered-reads.spec \ useGrid=0 scriptOnGrid=0 \ doFragmentCorrection=1 \ stopAfter=unitigger \ $wrk/CAcorrected/$pfx.frg else if [ ! -e "$wrk/CAcorrected/$pfx.qc" ] ; then perl $bin/runCA -p $pfx -d $wrk/CAcorrected -s $wrk/build-ordered-reads.spec \ useGrid=0 scriptOnGrid=0 \ doFragmentCorrection=1 \ $wrk/CAcorrected/$pfx.frg fi fi ######################################## # # Assembly with the perfectly mapping reads. # if [ -e $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq ] ; then if [ ! -d $wrk/CAordered ] ; then mkdir $wrk/CAordered fi if [ ! -d "$wrk/CAordered/$pfx.frg" ] ; then $bin/fastqToCA \ -libraryname L \ -technology sanger \ -type sanger \ -reads $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq \ > $wrk/CAordered/$pfx.frg fi if [ ! -d "$wrk/CAordered/$pfx.tigStore" ] ; then perl $bin/runCA -p $pfx -d $wrk/CAordered -s $wrk/build-ordered-reads.spec \ useGrid=0 scriptOnGrid=0 \ stopAfter=unitigger \ $wrk/CAordered/$pfx.frg else if [ ! -e "$wrk/CAordered/$pfx.qc" ] ; then perl $bin/runCA -p $pfx -d $wrk/CAordered -s $wrk/build-ordered-reads.spec \ useGrid=0 scriptOnGrid=0 \ $wrk/CAordered/$pfx.frg fi fi fi ######################################## # # Assembly with the perfectly mapping reads. # if [ -e $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq ] ; then if [ ! -d $wrk/CAorderedCorrected ] ; then mkdir $wrk/CAorderedCorrected fi if [ ! -d "$wrk/CAorderedCorrected/$pfx.frg" ] ; then $bin/fastqToCA \ -libraryname L \ -technology sanger \ -type sanger \ -reads $wrk/BO/$pfx.blasr.sam.coords.mapped.ordered.fastq \ > $wrk/CAorderedCorrected/$pfx.frg fi if [ ! -d "$wrk/CAorderedCorrected/$pfx.tigStore" ] ; then perl $bin/runCA -p $pfx -d $wrk/CAorderedCorrected -s $wrk/build-ordered-reads.spec \ useGrid=0 scriptOnGrid=0 \ doFragmentCorrection=1 \ stopAfter=unitigger \ $wrk/CAorderedCorrected/$pfx.frg else if [ ! -e "$wrk/CAorderedCorrected/$pfx.qc" ] ; then perl $bin/runCA -p $pfx -d $wrk/CAorderedCorrected -s $wrk/build-ordered-reads.spec \ useGrid=0 scriptOnGrid=0 \ doFragmentCorrection=1 \ $wrk/CAorderedCorrected/$pfx.frg fi fi fi canu-1.3/src/fastq-utilities/fastqAnalyze.C000066400000000000000000000347331272335422300210040ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_GKP/fastqAnalyze.C * * Modifications by: * * Brian P. Walenz from 2012-FEB-24 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2015-JAN-13 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-03 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include #include using namespace std; // Map a letter to an index in the freq arrays. uint32 baseToIndex[256]; char indexToBase[8]; #define FREQ_A 0 #define FREQ_C 1 #define FREQ_G 2 #define FREQ_T 3 #define FREQ_N 4 // Just N's #define FREQ_g 5 // Just -'s #define FREQ_Z 6 // Everything else #define FREQ_NUM 7 #define MAX_READ_LEN 1024 * 1024 class nucFreq { public: nucFreq() { memset(mono, 0, sizeof(uint64) * FREQ_NUM); memset(di, 0, sizeof(uint64) * FREQ_NUM * FREQ_NUM); memset(tri, 0, sizeof(uint64) * FREQ_NUM * FREQ_NUM * FREQ_NUM); }; uint64 mono[FREQ_NUM]; uint64 di[FREQ_NUM][FREQ_NUM]; uint64 tri[FREQ_NUM][FREQ_NUM][FREQ_NUM]; }; class nucOut { public: nucOut(char a, char b, char c, uint64 cnt, double frq) { label[0] = a; label[1] = b; label[2] = c; label[3] = 0; count = cnt; freq = frq; }; bool operator<(nucOut const &that) const { return(freq > that.freq); }; char label[4]; uint64 count; double freq; }; void doStats(char *inName, char *otName) { uint64 totSeqs = 0; uint64 totBases = 0; vector seqLen; nucFreq *freq = new nucFreq; char A[MAX_READ_LEN]; char B[MAX_READ_LEN]; char C[MAX_READ_LEN]; char D[MAX_READ_LEN]; errno = 0; FILE *F = fopen(inName, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", inName, strerror(errno)), exit(1); //errno = 0; //FILE *O = fopen(otName, "w"); //if (errno) // fprintf(stderr, "Failed to open '%s' for writing: %s\n", otName, strerror(errno)), exit(1); while (!feof(F)) { fgets(A, MAX_READ_LEN, F); fgets(B, MAX_READ_LEN, F); chomp(B); fgets(C, MAX_READ_LEN, F); fgets(D, MAX_READ_LEN, F); chomp(D); if ((A[0] != '@') || (C[0] != '+')) { fprintf(stderr, "WARNING: sequence isn't fastq.\n"); fprintf(stderr, "WARNING: %s", A); fprintf(stderr, "WARNING: %s\n", B); fprintf(stderr, "WARNING: %s", C); fprintf(stderr, "WARNING: %s\n", D); } uint32 a = 0; uint32 b = baseToIndex[B[0]]; uint32 c = baseToIndex[B[1]]; uint32 ii; freq->mono[b]++; freq->mono[c]++; freq->di[b][c]++; for (ii=2; B[ii]; ii++) { a = b; b = c; c = baseToIndex[B[ii]]; freq->mono[c]++; freq->di[b][c]++; freq->tri[a][b][c]++; } ii--; seqLen.push_back(ii); totSeqs++; totBases += ii; if ((totSeqs % 10000) == 0) fprintf(stderr, "Reading "F_U64"\r", totSeqs); } fprintf(stderr, "Read "F_U64"\n", totSeqs); fprintf(stdout, "%s\n", inName); fprintf(stdout, "\n"); fprintf(stdout, "sequences\t"F_U64"\n", totSeqs); fprintf(stdout, "bases\t"F_U64"\n", totBases); fprintf(stdout, "\n"); fprintf(stdout, "average\t"F_U64"\n", totBases / totSeqs); fprintf(stdout, "\n"); //sort(seqLen.begin(), seqLen.end()); uint64 min = UINT64_MAX; uint64 max = 0; for (uint32 ii=0; ii output; fprintf(stdout, "\n"); fprintf(stdout, "mononucleotide\n"); fprintf(stdout, "\n"); for (uint32 ii=0; iimono[ii] > 0) output.push_back(nucOut(indexToBase[ii], 0, 0, freq->mono[ii], freq->mono[ii] * 100.0 / totBases)); sort(output.begin(), output.end()); for (uint32 ii=0; iidi[ii][jj] > 0) output.push_back(nucOut(indexToBase[ii], indexToBase[jj], 0, freq->di[ii][jj], freq->di[ii][jj] * 100.0 / totBases)); sort(output.begin(), output.end()); for (uint32 ii=0; iitri[ii][jj][kk] > 0) output.push_back(nucOut(indexToBase[ii], indexToBase[jj], indexToBase[kk], freq->tri[ii][jj][kk], freq->tri[ii][jj][kk] * 100.0 / totBases)); sort(output.begin(), output.end()); for (uint32 ii=0; ii 0)) { fgets(A, MAX_READ_LEN, F); fgets(B, MAX_READ_LEN, F); fgets(C, MAX_READ_LEN, F); fgets(D, MAX_READ_LEN, F); chomp(D); for (uint32 x=0; D[x] != 0; x++) { if (D[x] < '!') isNotSanger = true; if (D[x] < ';') isNotSolexa = true; if (D[x] < '@') isNotIllumina3 = true; // Illumina 1.3 if (D[x] < 'B') isNotIllumina5 = true; // Illumina 1.5 if (D[x] < '!') isNotIllumina8 = true; // Illumina 1.5 if ('I' < D[x]) isNotSanger = true; if ('h' < D[x]) isNotSolexa = true; if ('h' < D[x]) isNotIllumina3 = true; // Illumina 1.3 if ('h' < D[x]) isNotIllumina5 = true; // Illumina 1.5 if ('J' < D[x]) isNotIllumina8 = true; // Illumina 1.5 qvCounts[D[x]]++; //fprintf(stderr, "%d%d%d%d%d %c %d\n", // isNotSanger, isNotSolexa, isNotIllumina3, isNotIllumina5, isNotIllumina8, D[x], x); } //fprintf(stderr, "%d%d%d%d%d '%s'\n", // isNotSanger, isNotSolexa, isNotIllumina3, isNotIllumina5, isNotIllumina8, D); numValid = 0; if (isNotSanger == false) numValid++; if (isNotSolexa == false) numValid++; if (isNotIllumina3 == false) numValid++; if (isNotIllumina5 == false) numValid++; if (isNotIllumina8 == false) numValid++; numTrials--; } fclose(F); fprintf(stdout, "%s --", inName); if (isNotSanger == false) fprintf(stdout, " SANGER"); if (isNotSolexa == false) fprintf(stdout, " SOLEXA"); if (isNotIllumina3 == false) fprintf(stdout, " ILLUMINA_1.3+"); if (isNotIllumina5 == false) fprintf(stdout, " ILLUMINA_1.5+"); if (isNotIllumina8 == false) fprintf(stdout, " ILLUMINA_1.8+"); if (numValid == 0) fprintf(stdout, " NO_VALID_ENCODING"); fprintf(stdout, "\n"); if (numValid == 0) { fprintf(stdout, "QV histogram:\n"); for (uint32 c=0, i=0; i<12; i++) { fprintf(stdout, "%3d: ", i * 10); for (uint32 j=0; j<10; j++, c++) fprintf(stdout, " %8d/%c", qvCounts[c], isprint(c) ? c : '.'); fprintf(stdout, "\n"); } } if (isNotSanger == false) originalIsSanger = true; if (isNotSolexa == false) originalIsSolexa = true; if (isNotIllumina3 == false) originalIsIllumina = true; if (isNotIllumina5 == false) originalIsIllumina = true; if (isNotIllumina8 == false) originalIsSanger = true; } void doTransformQV(char *inName, char *otName, bool originalIsSolexa, bool originalIsIllumina, bool originalIsSanger) { uint32 numValid = 0; char A[MAX_READ_LEN]; char B[MAX_READ_LEN]; char C[MAX_READ_LEN]; char D[MAX_READ_LEN]; if (originalIsSolexa == true) numValid++; if (originalIsIllumina == true) numValid++; if (originalIsSanger == true) numValid++; if (numValid == 0) fprintf(stderr, "No QV decision made. No valid encoding found. Specify a QV encoding to convert from.\n"), exit(0); if (numValid > 1) fprintf(stderr, "No QV decision made. Multiple valid encodings found. Specify a QV encoding to convert from.\n"), exit(0); if (originalIsSanger == true) fprintf(stderr, "No QV changes needed; original is in sanger format already.\n"), exit(0); errno = 0; FILE *F = fopen(inName, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", inName, strerror(errno)), exit(1); errno = 0; FILE *O = fopen(otName, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", otName, strerror(errno)), exit(1); while (!feof(F)) { fgets(A, MAX_READ_LEN, F); fgets(B, MAX_READ_LEN, F); fgets(C, MAX_READ_LEN, F); fgets(D, MAX_READ_LEN, F); chomp(D); if (feof(F)) break; for (uint32 x=0; D[x] != 0; x++) { if (originalIsSolexa) { double qs = D[x] - '@'; qs /= 10.0; qs = 10.0 * log10(pow(10.0, qs) + 1); D[x] = lround(qs) + '0'; } if (originalIsIllumina) { D[x] -= '@'; D[x] += '!'; } } fprintf(O, "%s%s%s%s\n", A, B, C, D); } fclose(F); fclose(O); } int main(int argc, char **argv) { char *inName = NULL; char *otName = NULL; bool originalIsSolexa = false; bool originalIsIllumina = false; bool originalIsSanger = false; bool convertToSanger = false; bool computeStats = false; int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-o") == 0) { otName = argv[++arg]; convertToSanger = true; } else if (strcmp(argv[arg], "-solexa") == 0) { originalIsSolexa = true; } else if (strcmp(argv[arg], "-illumina") == 0) { originalIsIllumina = true; } else if (strcmp(argv[arg], "-sanger") == 0) { originalIsSanger = true; } else if (strcmp(argv[arg], "-stats") == 0) { computeStats = true; } else if (inName == NULL) { inName = argv[arg]; } else { err++; } arg++; } if ((err) || (inName == NULL)) { fprintf(stderr, "usage: %s [-stats] [-o output.fastq] input.fastq\n", argv[0]); fprintf(stderr, " If no options are given, input.fastq is analyzed and a best guess for the\n"); fprintf(stderr, " QV encoding is output. Otherwise, the QV encoding is converted to Sanger-style\n"); fprintf(stderr, " using this guess.\n"); fprintf(stderr, "\n"); fprintf(stderr, " In some cases, the encoding cannot be determined. When this occurs, no guess is\n"); fprintf(stderr, " output. For conversion, you can force the input QV type with:\n"); fprintf(stderr, "\n"); fprintf(stderr, " -solexa input QV is solexa\n"); fprintf(stderr, " -illumina input QV is illumina\n"); fprintf(stderr, " -sanger input QV is sanger\n"); fprintf(stderr, "\n"); fprintf(stderr, " -o sanger-style-output.fastq\n"); fprintf(stderr, "\n"); fprintf(stderr, " If -stats is supplied, no QV analysis or conversion is performed, but some simple\n"); fprintf(stderr, " statistics are computed and output to stdout.\n"); fprintf(stderr, "\n"); exit(1); } for (uint32 ii=0; ii<256; ii++) baseToIndex[ii] = FREQ_Z; baseToIndex['a'] = FREQ_A; baseToIndex['A'] = FREQ_A; baseToIndex['c'] = FREQ_C; baseToIndex['C'] = FREQ_C; baseToIndex['g'] = FREQ_G; baseToIndex['G'] = FREQ_G; baseToIndex['t'] = FREQ_T; baseToIndex['T'] = FREQ_T; baseToIndex['n'] = FREQ_N; baseToIndex['N'] = FREQ_N; baseToIndex['-'] = FREQ_g; baseToIndex['-'] = FREQ_g; indexToBase[FREQ_A] = 'A'; indexToBase[FREQ_C] = 'C'; indexToBase[FREQ_G] = 'G'; indexToBase[FREQ_T] = 'T'; indexToBase[FREQ_N] = 'N'; indexToBase[FREQ_g] = '-'; indexToBase[FREQ_Z] = '?'; if (computeStats) doStats(inName, otName), exit(0); doAnalyzeQV(inName, originalIsSolexa, originalIsIllumina, originalIsSanger); if (convertToSanger) doTransformQV(inName, otName, originalIsSolexa, originalIsIllumina, originalIsSanger); exit(0); } canu-1.3/src/fastq-utilities/fastqAnalyze.mk000066400000000000000000000007311272335422300212200ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := fastqAnalyze SOURCES := fastqAnalyze.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/fastq-utilities/fastqSample.C000066400000000000000000000361011272335422300206110ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_GKP/fastqSample.C * * Modifications by: * * Brian P. Walenz from 2010-FEB-22 to 2013-AUG-01 * are Copyright 2010-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-AUG-06 to 2015-FEB-24 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include #include using namespace std; #define MAXLEN 1024*1024 class aRead { public: aRead() { memset(a, 0, sizeof(char) * MAXLEN); memset(b, 0, sizeof(char) * MAXLEN); memset(c, 0, sizeof(char) * MAXLEN); memset(d, 0, sizeof(char) * MAXLEN); }; bool read(FILE *F) { a[0] = 0; b[0] = 0; c[0] = 0; d[0] = 0; if (F == NULL) return(false); fgets(a, MAXLEN, F); fgets(b, MAXLEN, F); fgets(c, MAXLEN, F); fgets(d, MAXLEN, F); if (feof(F) == true) return(false); if ((a[0] != '@') || (c[0] != '+')) { fprintf(stderr, "ERROR: Not FastQ. Read lines:\n"); fprintf(stderr, " %s", a); fprintf(stderr, " %s", b); fprintf(stderr, " %s", c); fprintf(stderr, " %s", d); exit(1); } return(true); }; void write(FILE *F) { if (F == NULL) return; fputs(a, F); fputs(b, F); fputs(c, F); fputs(d, F); }; uint32 length(void) { return(strlen(b) - 1); // Newline still exists }; private: char a[MAXLEN]; char b[MAXLEN]; char c[MAXLEN]; char d[MAXLEN]; }; class anInput { public: anInput() { id = 0; len = 0; }; anInput(uint64 id_, uint32 len1_, uint32 len2_) { id = id_; len = len1_ + len2_; }; uint64 id; uint32 len; }; inline bool anInputByLongest(const anInput &a, const anInput &b) { return(a.len > b.len); }; int main(int argc, char **argv) { aRead *Ar = new aRead, *Br = new aRead; FILE *Ai = NULL, *Bi = NULL; FILE *Ao = NULL, *Bo = NULL; vector ids; vector sav; char *INPNAME = NULL; char *OUTNAME = NULL; bool AUTONAME = false; uint64 NUMINPUT = 0; // Number of pairs in the input uint64 READLENGTH = 0; // For mated reads, 2x read size bool isMated = true; uint32 MINLEN = 0; uint32 LONGEST = false; uint64 GENOMESIZE = 0; // Size of the genome in bp double COVERAGE = 0; // Desired coverage in output uint64 NUMOUTPUT = 0; // Number of pairs to output double FRACTION = 0.0; // Desired fraction of the input uint64 BASES = 0; // Desired amount of sequence char path1[FILENAME_MAX]; char path2[FILENAME_MAX]; srand48(time(NULL)); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-I") == 0) { INPNAME = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { OUTNAME = argv[++arg]; } else if (strcmp(argv[arg], "-A") == 0) { AUTONAME = true; } else if (strcmp(argv[arg], "-T") == 0) { NUMINPUT = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-L") == 0) { READLENGTH = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-U") == 0) { isMated = false; } else if (strcmp(argv[arg], "-m") == 0) { MINLEN = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-max") == 0) { LONGEST = true; } else if (strcmp(argv[arg], "-g") == 0) { GENOMESIZE = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-c") == 0) { COVERAGE = atof(argv[++arg]); } else if (strcmp(argv[arg], "-p") == 0) { NUMOUTPUT = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-f") == 0) { FRACTION = atof(argv[++arg]); } else if (strcmp(argv[arg], "-b") == 0) { BASES = atoi(argv[++arg]); } else { err++; } arg++; } if (INPNAME == NULL) err++; if (OUTNAME == NULL) OUTNAME = INPNAME; if ((GENOMESIZE == 0) && (COVERAGE != 0)) err++; if ((COVERAGE == 0) && (NUMOUTPUT == 0) && (FRACTION == 0.0) && (BASES == 0)) err++; if (err) { fprintf(stderr, "\n"); fprintf(stderr, "usage: %s [opts]\n", argv[0]); fprintf(stderr, " Input Specification\n"); fprintf(stderr, " -I NAME input name (prefix) of the reads\n"); fprintf(stderr, " -T T total number of mate pairs in the input (if not supplied, will be counted)\n"); fprintf(stderr, " -L L length of a single read (if not supplied, will be determined)\n"); fprintf(stderr, " -U reads are unmated, expected in *.u.fastq\n"); fprintf(stderr, "\n"); fprintf(stderr, " Output Specification\n"); fprintf(stderr, " -O NAME output name (prefix) of the reads (default is same as -I)\n"); fprintf(stderr, " -A automatically include coverage or number of reads in the output name\n"); fprintf(stderr, " -m L ignore reads shorter than L bases\n"); fprintf(stderr, " -max don't sample randomly, pick the longest reads\n"); fprintf(stderr, "\n"); fprintf(stderr, " Method 1: specify desired output coverage:\n"); fprintf(stderr, " -g G genome size\n"); fprintf(stderr, " -c C desired coverage in the output reads\n"); fprintf(stderr, "\n"); fprintf(stderr, " Method 2: specify desired number of output pairs\n"); fprintf(stderr, " -p N for mated reads, output 2N reads, or N pairs of reads\n"); fprintf(stderr, " for unmated reads, output N reads\n"); fprintf(stderr, "\n"); fprintf(stderr, " Method 3: specify a desired fraction of the input:\n"); fprintf(stderr, " -f F output F * T pairs of reads (T as above in -t option)\n"); fprintf(stderr, " 0.0 < F <= 1.0\n"); fprintf(stderr, "\n"); fprintf(stderr, " Method 4: specify a desired total length\n"); fprintf(stderr, " -b B output reads/pairs until B bases is exceeded\n"); fprintf(stderr, "\n"); fprintf(stderr, "\n"); fprintf(stderr, "Samples reads from paired Illumina reads NAME.1.fastq and NAME.2.fastq and outputs:\n"); fprintf(stderr, " NAME.Cx.1.fastq and N.Cx.2.fastq (for coverage based sampling)\n"); fprintf(stderr, " NAME.n=N.1.fastq and N.n=N.2.fastq (for coverage based sampling)\n"); fprintf(stderr, "\n"); fprintf(stderr, "If -T is not supplied, the number of reads will be counted for you.\n"); fprintf(stderr, "\n"); if (INPNAME == NULL) fprintf(stderr, "ERROR: no name supplied with -I.\n"); if ((GENOMESIZE == 0) && (COVERAGE != 0)) fprintf(stderr, "ERROR: no genome size supplied with -g (when using -c)\n"); if ((COVERAGE == 0) && (NUMOUTPUT == 0) && (FRACTION == 0.0) && (BASES == 0)) fprintf(stderr, "ERROR: no method supplied with -c, -p, -f or -b\n"); fprintf(stderr, "\n"); exit(1); } // // We know not enough about the reads, and are forced to scan the entire // inputs. // uint64 totBasesInInput = 0; uint64 totPairsInInput = 0; if ((NUMINPUT == 0) || (READLENGTH == 0)) { uint64 Ac = 0; uint64 Bc = 0; fprintf(stderr, "Counting the number of reads in the input.\n"); sprintf(path1, "%s.%c.fastq", INPNAME, (isMated == true) ? '1' : 'u'); sprintf(path2, "%s.%c.fastq", INPNAME, (isMated == true) ? '2' : 'u'); errno = 0; Ai = fopen(path1, "r"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", path1, strerror(errno)), exit(1); if (isMated == true) { errno = 0; Bi = fopen(path2, "r"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", path2, strerror(errno)), exit(1); } bool moreA = Ar->read(Ai); bool moreB = Br->read(Bi); while (moreA || moreB) { uint32 lA = Ar->length(); uint32 lB = Br->length(); if (lA > 0) Ac++; if (lB > 0) Bc++; ids.push_back(anInput(totPairsInInput, lA, lB)); sav.push_back(false); totPairsInInput += 1; totBasesInInput += lA + lB; moreA = Ar->read(Ai); moreB = Br->read(Bi); } if (Ai) fclose(Ai); if (Bi) fclose(Bi); fprintf(stderr, "Found "F_U64" bases and "F_U64" reads in '%s'\n", totBasesInInput, totPairsInInput, path1); if (Ac != Bc) { fprintf(stderr, "ERROR: Number of reads in the .1 and .2 files must be the same.\n"); exit(1); } } // Otherwise, both NUMINPUT and READLENGTH are defined, so we can fill out ids // with defaults. else { if (isMated == false) for (uint64 ii=0; ii 0) { nBasesToOutput = (uint64)(COVERAGE * GENOMESIZE); } if (NUMOUTPUT > 0) { nPairsToOutput = NUMOUTPUT; } if (FRACTION > 0) { nPairsToOutput = (uint64)(FRACTION * totPairsInInput); } if (BASES > 0) { nBasesToOutput = BASES; } if (totBasesInInput < nBasesToOutput) fprintf(stderr, "ERROR: not enough reads, "F_U64" bp in input, "F_U64" needed for desired .....\n", totBasesInInput, nBasesToOutput), exit(1); if (totPairsInInput < nPairsToOutput) fprintf(stderr, "ERROR: not enough reads, "F_U64" %s in input, "F_U64" needed for desired ......\n", totPairsInInput, (isMated) ? "pairs" : "reads", nPairsToOutput), exit(1); //fprintf(stderr, "OUTPUT: %lu bases\n", nBasesToOutput); //fprintf(stderr, "OUTPUT: %lu pairs\n", nPairsToOutput); // // Randomize the ID list, or sort by length. // if (LONGEST) { fprintf(stderr, "Sorting by length\n"); sort(ids.begin(), ids.end(), anInputByLongest); } else { fprintf(stderr, "Shuffling sequences\n"); for (uint64 i=0; i 0) { uint64 nPairs = 0; for (uint64 i=0; i 0) { uint64 nBases = 0; for (uint64 i=0; i 0) { sprintf(path1, "%s.x=%07.3f.n=%09"F_U64P".%c.fastq", OUTNAME, (double)nBasesToOutput / GENOMESIZE, nPairsToOutput, (isMated == true) ? '1' : 'u'); sprintf(path2, "%s.x=%07.3f.n=%09"F_U64P".%c.fastq", OUTNAME, (double)nBasesToOutput / GENOMESIZE, nPairsToOutput, (isMated == true) ? '2' : 'u'); } else { sprintf(path1, "%s.x=UNKNOWN.n=%09"F_U64P".%c.fastq", OUTNAME, nPairsToOutput, (isMated == true) ? '1' : 'u'); sprintf(path2, "%s.x=UNKNOWN.n=%09"F_U64P".%c.fastq", OUTNAME, nPairsToOutput, (isMated == true) ? '2' : 'u'); } errno = 0; Ao = fopen(path1, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", path1, strerror(errno)), exit(1); if (isMated == true) { errno = 0; Bo = fopen(path2, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", path2, strerror(errno)), exit(1); } uint64 i=0; uint64 s=0; if (isMated == true) { if (nPairsToOutput > 0) fprintf(stderr, "Extracting "F_U64" mate pairs into %s and %s\n", nPairsToOutput, path1, path2); else fprintf(stderr, "Extracting "F_U64" bases of mate pairs into %s and %s\n", nBasesToOutput, path1, path2); for (; Ar->read(Ai) && Br->read(Bi); i++) { if ((i < totPairsInInput) && (sav[i])) { Ar->write(Ao); Br->write(Bo); s++; } } fclose(Ai); fclose(Bi); fclose(Ao); fclose(Bo); } else { if (nPairsToOutput > 0) fprintf(stderr, "Extracting "F_U64" reads into %s\n", nPairsToOutput, path1); else fprintf(stderr, "Extracting "F_U64" bases of reads into %s\n", nBasesToOutput, path1); for (; Ar->read(Ai); i++) { if ((i < totPairsInInput) && (sav[i])) { Ar->write(Ao); s++; } } fclose(Ai); fclose(Ao); } delete Ar; delete Br; if (i > totPairsInInput) { fprintf(stderr, "WARNING: There are "F_U64" %s in the input; you claimed there are "F_U64" (-t option) %s.\n", i, (isMated) ? "mates" : "reads", totPairsInInput, (isMated) ? "mates" : "reads"); fprintf(stderr, "WARNING: Result is not a random sample of the input file.\n"); } if (i < totPairsInInput) { fprintf(stderr, "WARNING: There are "F_U64" %s in the input; you claimed there are "F_U64" (-t option) %s.\n", i, (isMated) ? "mates" : "reads", totPairsInInput, (isMated) ? "mates" : "reads"); fprintf(stderr, "WARNING: Result is only %f X coverage.\n", (double)s * READLENGTH / GENOMESIZE); } return(0); } canu-1.3/src/fastq-utilities/fastqSample.mk000066400000000000000000000007271272335422300210430ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := fastqSample SOURCES := fastqSample.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/fastq-utilities/fastqSimulate-checkCoverage.pl000066400000000000000000000104561272335422300241400ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/AS_GKP/fastqSimulate-checkCoverage.pl # # Modifications by: # # Brian P. Walenz on 2014-FEB-20 # are Copyright 2014 J. Craig Venter Institute, and # are subject to the GNU General Public License version 2 # # Brian P. Walenz on 2015-JAN-13 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; system("leaff -G 1 2000 2000 > base.fasta"); system("leaff -H -G 1 500 500 | tr ACGT NNNN >> base.fasta"); system("leaff -H -G 1 2000 2000 >> base.fasta"); my (@rcov1, @ccov1); my (@rcov2, @ccov2); my (@rcov3, @ccov3); if (! -e "cov") { my $max = -s "base.fasta"; my $maxiter = 100; for (my $iter=0; $iter<$maxiter; $iter++) { system("fastqSimulate -em 0 -ei 0 -ed 0 -f base.fasta -o t1 -l 100 -x 100 -pe 1000 0 > /dev/null 2>&1"); system("fastqSimulate -em 0 -ei 0 -ed 0 -f base.fasta -o t2 -l 100 -x 100 -allowgaps -pe 1000 0 > /dev/null 2>&1"); system("fastqSimulate -em 0 -ei 0 -ed 0 -f base.fasta -o t3 -l 100 -x 100 -allowgaps -allowns -pe 1000 0 > /dev/null 2>&1"); system("rm *.1.fastq *.2.fastq *.c.fastq"); print STDERR "Summarizing for iter $iter\n"; open(IN, "< t1.i.fastq"); summarize(\@rcov1, \@ccov1); close(IN); open(IN, "< t2.i.fastq"); summarize(\@rcov2, \@ccov2); close(IN); open(IN, "< t3.i.fastq"); summarize(\@rcov3, \@ccov3); close(IN); } open(COV, "> cov"); for (my $x=0; $x<$max; $x++) { $rcov1[$x] += 0; $rcov1[$x] /= $maxiter; $ccov1[$x] += 0; $ccov1[$x] /= $maxiter; $rcov2[$x] += 0; $rcov2[$x] /= $maxiter; $ccov2[$x] += 0; $ccov2[$x] /= $maxiter; $rcov3[$x] += 0; $rcov3[$x] /= $maxiter; $ccov3[$x] += 0; $ccov3[$x] /= $maxiter; print COV "$x\t$rcov1[$x]\t$ccov1[$x]\t$rcov2[$x]\t$ccov2[$x]\t$rcov3[$x]\t$ccov3[$x]\n"; } close(COV); } open(GP, "> cov.gp"); print GP "set terminal png size 1280,800\n"; print GP "set output 'cov.png'\n"; print GP "plot 'cov' using 1:2 with lines lt 1 lw 1 title 'read cov, no Ns', \\\n"; print GP " 'cov' using 1:4 with lines lt 2 lw 1 title 'read cov, clone span Ns', \\\n"; print GP " 'cov' using 1:6 with lines lt 3 lw 1 title 'read cov, both span Ns', \\\n"; print GP " 'cov' using 1:3 with lines lt 1 lw 2 title 'clone cov, no Ns', \\\n"; print GP " 'cov' using 1:5 with lines lt 2 lw 2 title 'clone cov, clone span Ns', \\\n"; print GP " 'cov' using 1:7 with lines lt 3 lw 2 title 'clone cov, both span Ns'\n"; close(GP); system("gnuplot cov.gp"); sub summarize(@@) { my ($rcov, $ccov) = @_; while (!eof(IN)) { my $a = ; chomp $a; my $b = ; chomp $b; my $l = length($b); my $c = ; my $d = ; my $bgn; my $end; my $spn; if ($a =~ m/^\@PE_\d+_\d+\@(\d+)-(\d+)\/1/) { $bgn = $1; $end = $1 + $l; $spn = $2; } if ($a =~ m/^\@PE_\d+_\d+\@(\d+)-(\d+)\/2/) { $bgn = $2 - $l; $end = $2; $spn = $bgn; # No clone span for the second read } die if (!defined($bgn)); for (my $x=$bgn; $x<$end; $x++) { ${$rcov}[$x]++; } for (my $x=$bgn; $x<$spn; $x++) { ${$ccov}[$x]++; } } } canu-1.3/src/fastq-utilities/fastqSimulate-perfectSep.pl000066400000000000000000000235401272335422300235050ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/AS_GKP/fastqSimulate-perfectSep.pl # # Modifications by: # # Brian P. Walenz from 2011-DEC-28 to 2013-AUG-01 # are Copyright 2011,2013 J. Craig Venter Institute, and # are subject to the GNU General Public License version 2 # # Brian P. Walenz on 2015-JAN-13 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Build MP, PE and 454 reads for a reference sequence. The MP reads are filtered into # a perfectly classified set with two libraries PEperfect and MPperfect. Chimers # can be rescued or deleted. my $prefix = shift @ARGV; my $reference = shift @ARGV; my $error = "0.01"; my $BBcoverage = "08"; my $PEcoverage = "25"; my $PEinsert = "0500"; my $PEstddev = "050"; my $MPcoverage = "10"; my $MPinsert = "3000"; my $MPstddev = "300"; my $MPenrich = "0.8"; my $BBreadLen = "400"; my $PEreadLen = "150"; my $MPreadLen = "150"; my $name; # # BUILD 454 # $name = "$prefix.${BBreadLen}bp.fragment.${BBcoverage}x"; if (! -e "$name.s.fastq") { system("fastqSimulate -f $reference -o $name -l $BBreadLen -x $BBcoverage -e $error -se"); unlink("$name.frg"); } if (! -e "$name.frg") { system("fastqToCA -libraryname BB -type sanger -reads $name.s.fastq > $name.frg"); } # # BUILD PE # $name = "$prefix.${PEreadLen}bp.${PEinsert}bpPE.${PEcoverage}x"; if (! -e "$name.i.fastq") { system("fastqSimulate -f $reference -o $name -l $PEreadLen -x $PEcoverage -e $error -pe $PEinsert $PEstddev"); unlink("$name.frg"); } if (! -e "$name.frg") { system("fastqToCA -libraryname BB -insertsize $PEinsert $PEstddev -innie -type sanger -reads $name.1.fastq,$name.2.fastq > $name.frg"); } # # BUILD MP # $name = "$prefix.${MPreadLen}bp.${MPinsert}bpMP.${MPcoverage}x"; if (! -e "$name.i.fastq") { print STDERR "fastqSimulate -f $reference -o $name -l $MPreadLen -x $MPcoverage -e $error -mp $MPinsert $MPstddev $PEinsert $PEstddev $MPenrich\n"; system("fastqSimulate -f $reference -o $name -l $MPreadLen -x $MPcoverage -e $error -mp $MPinsert $MPstddev $PEinsert $PEstddev $MPenrich"); unlink("$name.frg"); } if (! -e "$name.frg") { system("fastqToCA -libraryname MP -insertsize $MPinsert $MPstddev -outtie -type sanger -reads $name.1.fastq,$name.2.fastq > $name.frg"); } # # FILTER MP # open(tMP1, "> $name.tMP.1.fastq"); open(tMP2, "> $name.tMP.2.fastq"); open(tMPi, "> $name.tMP.i.fastq"); open(fPE1, "> $name.fPE.1.fastq"); open(fPE2, "> $name.fPE.2.fastq"); open(fPEi, "> $name.fPE.i.fastq"); open(aPE1, "> $name.aPE.1.fastq"); # A-junction reads that are PE open(aPE2, "> $name.aPE.2.fastq"); open(aPEi, "> $name.aPE.i.fastq"); open(aMP1, "> $name.aMP.1.fastq"); # A-junction reads that are MP open(aMP2, "> $name.aMP.2.fastq"); open(aMPi, "> $name.aMP.i.fastq"); open(bPE1, "> $name.bPE.1.fastq"); # B-junction reads that are PE open(bPE2, "> $name.bPE.2.fastq"); open(bPEi, "> $name.bPE.i.fastq"); open(bMP1, "> $name.bMP.1.fastq"); # B-junction reads that are MP open(bMP2, "> $name.bMP.2.fastq"); open(bMPi, "> $name.bMP.i.fastq"); open(MP1, "< $name.1.fastq"); open(MP2, "< $name.2.fastq"); my ($a1, $b1, $c1, $d1); my ($a2, $b2, $c2, $d2); while (!eof(MP1) && !eof(MP2)) { $a1 = ; chomp $a1; $a2 = ; chomp $a2; # Painful. The reverse()s below also reverse the newline. $b1 = ; chomp $b1; $b2 = ; chomp $b2; $c1 = ; chomp $c1; $c2 = ; chomp $c2; $d1 = ; chomp $d1; $d2 = ; chomp $d2; if ($a1 =~ m/^.(...)_/) { my $typ = $1; if ($typ eq "tMP") { $b1 = reverse($b1); $b1 =~ tr/ACGTacgt/TGCAtgca/; $d1 = reverse($d1); $b2 = reverse($b2); $b2 =~ tr/ACGTacgt/TGCAtgca/; $d2 = reverse($d2); print tMP1 "$a1\n$b1\n$c1\n$d1\n"; print tMP2 "$a2\n$b2\n$c2\n$d2\n"; print tMPi "$a1\n$b1\n$c1\n$d1\n"; print tMPi "$a2\n$b2\n$c2\n$d2\n"; } elsif ($typ eq "fPE") { print fPE1 "$a1\n$b1\n$c1\n$d1\n"; print fPE2 "$a2\n$b2\n$c2\n$d2\n"; print fPEi "$a1\n$b1\n$c1\n$d1\n"; print fPEi "$a2\n$b2\n$c2\n$d2\n"; } elsif ($typ eq "aMP") { if ($a1 =~ m/^...._\d+_\d+.\d+-\d+_(\d+)\/(\d+)\/\d+\/\d$/) { my $len = $2; my $pos = $1; if ($pos < $MPreadLen / 2) { # Saving the 5' end, makes MP. $b1 = substr($b1, 0, $pos); $b1 = reverse($b1); $b1 =~ tr/ACGTacgt/TGCAtgca/; $d1 = substr($d1, 0, $pos); $d1 = reverse($d1); $b2 = reverse($b2); $b2 =~ tr/ACGTacgt/TGCAtgca/; $d2 = reverse($d2); print aMP1 "$a1\n$b1\n$c1\n$d1\n"; print aMP2 "$a2\n$b2\n$c2\n$d2\n"; print aMPi "$a1\n$b1\n$c1\n$d1\n"; print aMPi "$a2\n$b2\n$c2\n$d2\n"; } else { # Saving the 3' end, makes PE. $b1 = substr($b1, $pos); $d1 = substr($d1, $pos); print aPE1 "$a1\n$b1\n$c1\n$d1\n"; print aPE2 "$a2\n$b2\n$c2\n$d2\n"; print aPEi "$a1\n$b1\n$c1\n$d1\n"; print aPEi "$a2\n$b2\n$c2\n$d2\n"; } } else { chomp $a1; die "No aMP '$a1'\n"; } } elsif ($typ eq "bMP") { if ($a1 =~ m/^...._\d+_\d+.\d+-\d+_(\d+)\/(\d+)\/\d+\/\d$/) { my $pos = $1; my $len = $2; if ($pos < $MPreadLen / 2) { # Saving the 5' end, makes MP. $b1 = reverse($b1); $b1 =~ tr/ACGTacgt/TGCAtgca/; $d1 = reverse($d1); $b2 = substr($b2, 0, $pos); $b2 = reverse($b2); $b2 =~ tr/ACGTacgt/TGCAtgca/; $d2 = substr($d2, 0, $pos); $d2 = reverse($d2); print bMP1 "$a1\n$b1\n$c1\n$d1\n"; print bMP2 "$a2\n$b2\n$c2\n$d2\n"; print bMPi "$a1\n$b1\n$c1\n$d1\n"; print bMPi "$a2\n$b2\n$c2\n$d2\n"; } else { # Saving the 3' end, makes PE. $b2 = substr($b2, $pos); $d2 = substr($d2, $pos); print bPE1 "$a1\n$b1\n$c1\n$d1\n"; print bPE2 "$a2\n$b2\n$c2\n$d2\n"; print bPEi "$a1\n$b1\n$c1\n$d1\n"; print bPEi "$a2\n$b2\n$c2\n$d2\n"; } } else { chomp $a1; die "No bMP '$a1'\n"; } } elsif ($typ eq "cMP") { chomp; print STDERR "$a1\n"; } else { die "Unknown type '$typ'\n"; } } else { chomp $a1; die "no match: '$a1'\n"; } } close(tMP1); close(tMP2); close(tMPi); close(fPE1); close(fPE2); close(fPEi); close(aMP1); close(aMP2); close(aMPi); close(aPE1); close(aPE2); close(aPEi); close(bMP1); close(bMP2); close(bMPi); close(bPE1); close(bPE2); close(bPEi); system("fastqToCA -libraryname tMP -insertsize $MPinsert $MPstddev -innie -type sanger -mates $name.tMP.1.fastq,$name.tMP.2.fastq > $name.tMP.frg"); system("fastqToCA -libraryname fPE -insertsize $PEinsert $PEstddev -innie -type sanger -mates $name.fPE.1.fastq,$name.fPE.2.fastq > $name.fPE.frg"); system("fastqToCA -libraryname aMP -insertsize $MPinsert $MPstddev -innie -type sanger -mates $name.aMP.1.fastq,$name.aMP.2.fastq > $name.aMP.frg"); system("fastqToCA -libraryname aPE -insertsize $PEinsert $PEstddev -innie -type sanger -mates $name.aPE.1.fastq,$name.aPE.2.fastq > $name.aPE.frg"); system("fastqToCA -libraryname bMP -insertsize $MPinsert $MPstddev -innie -type sanger -mates $name.bMP.1.fastq,$name.bMP.2.fastq > $name.bMP.frg"); system("fastqToCA -libraryname bPE -insertsize $PEinsert $PEstddev -innie -type sanger -mates $name.bPE.1.fastq,$name.bPE.2.fastq > $name.bPE.frg"); system("perl /work/FRAGS/map-illumina-pairs.pl plot-tMP JUNK.150bp.3000bpMP.10x.tMP.1.fastq JUNK.150bp.3000bpMP.10x.tMP.2.fastq Yersinia_pestis_KIM10_AE009952.fasta"); system("perl /work/FRAGS/map-illumina-pairs.pl plot-fPE JUNK.150bp.3000bpMP.10x.fPE.1.fastq JUNK.150bp.3000bpMP.10x.fPE.2.fastq Yersinia_pestis_KIM10_AE009952.fasta"); system("perl /work/FRAGS/map-illumina-pairs.pl plot-aMP JUNK.150bp.3000bpMP.10x aMP.1.fastq JUNK.150bp.3000bpMP.10x.aMP.2.fastq Yersinia_pestis_KIM10_AE009952.fasta"); system("perl /work/FRAGS/map-illumina-pairs.pl plot-aPE JUNK.150bp.3000bpMP.10x.aPE.1.fastq JUNK.150bp.3000bpMP.10x.aPE.2.fastq Yersinia_pestis_KIM10_AE009952.fasta"); system("perl /work/FRAGS/map-illumina-pairs.pl plot-bMP JUNK.150bp.3000bpMP.10x.bMP.1.fastq JUNK.150bp.3000bpMP.10x.bMP.2.fastq Yersinia_pestis_KIM10_AE009952.fasta"); system("perl /work/FRAGS/map-illumina-pairs.pl plot-bPE JUNK.150bp.3000bpMP.10x.bPE.1.fastq JUNK.150bp.3000bpMP.10x.bPE.2.fastq Yersinia_pestis_KIM10_AE009952.fasta"); canu-1.3/src/fastq-utilities/fastqSimulate-sort.C000066400000000000000000000141051272335422300221400ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_GKP/fastqSimulate-sort.C * * Modifications by: * * Brian P. Walenz from 2013-MAR-21 to 2013-AUG-01 * are Copyright 2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2015-JAN-13 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include #include "AS_global.H" #include "AS_UTL_fileIO.H" #include "AS_UTL_reverseComplement.H" #include #include using namespace std; class pairedRead { public: bool operator<(const pairedRead &that) const { return((seqId < that.seqId) || ((seqId == that.seqId) && (seqPos < that.seqPos))); } uint32 seqId; uint32 seqPos; char *readA; char *readB; }; uint32 hLen = 1024; uint32 sLen = 1024 * 1024 * 16; char *a = NULL; char *b = NULL; char *c = NULL; char *d = NULL; char * readRead(FILE *inFile, uint32 &seq, uint32 &bgn, uint32 &end) { seq = 0; bgn = 0; end = 0; if (inFile == NULL) return(NULL); if (a == NULL) { a = new char [hLen]; b = new char [sLen]; c = new char [hLen]; d = new char [sLen]; } uint32 al = 0; uint32 bl = 0; uint32 cl = 0; uint32 dl = 0; fgets(a, hLen, inFile); al = strlen(a); fgets(b, sLen, inFile); bl = strlen(b); fgets(c, hLen, inFile); cl = strlen(c); fgets(d, sLen, inFile); dl = strlen(d); assert(a[0] == '@'); assert(c[0] == '+'); uint32 p=0; // @tMP_0_0@2569105-2572074_239/553/2571835/1 while (a[p] != '_') p++; p++; while (a[p] != '_') p++; p++; seq = strtoul(a+p, NULL, 10); while (a[p] != '@') p++; p++; bgn = strtoul(a+p, NULL, 10); while (a[p] != '-') p++; p++; end = strtoul(a+p, NULL, 10); //fprintf(stderr, "seq="F_U32" bgn="F_U32" end="F_U32" line %s", // seq, bgn, end, a); char *retstr = new char [al + bl + cl + dl + 1]; memcpy(retstr, a, al); memcpy(retstr + al, b, bl); memcpy(retstr + al + bl, c, cl); memcpy(retstr + al + bl + cl, d, dl); retstr[al + bl + cl + dl] = 0; return(retstr); } int main(int argc, char **argv) { char *inName1 = NULL; char *inName2 = NULL; char *otName1 = NULL; char *otName2 = NULL; int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-i1") == 0) { inName1 = argv[++arg]; } else if (strcmp(argv[arg], "-i2") == 0) { inName2 = argv[++arg]; } else if (strcmp(argv[arg], "-o1") == 0) { otName1 = argv[++arg]; } else if (strcmp(argv[arg], "-o2") == 0) { otName2 = argv[++arg]; } else { err++; } arg++; } if (inName1 == NULL) err++; if (otName1 == NULL) err++; if ((inName2 == NULL) != (otName2 == NULL)) err++; if (err) { fprintf(stderr, "usage: %s -i1 in.1.fastq [-i2 in.2.fastq] -o1 out.1.fastq [-o2 out.2.fastq]\n", argv[0]); if (inName1 == NULL) fprintf(stderr, "ERROR: No in.1.fastq supplied with -i1.\n"); if (otName1 == NULL) fprintf(stderr, "ERROR: No out.1.fastq supplied with -i1.\n"); if ((inName2 == NULL) != (otName2 == NULL)) fprintf(stderr, "ERROR: Matedness of input and output don't agree (neither or both -i2 and -o2 must be used).\n"); exit(1); } errno = 0; FILE *inFile1 = (inName1 == NULL) ? NULL : fopen(inName1, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", inName1, strerror(errno)), exit(1); errno = 0; FILE *inFile2 = (inName2 == NULL) ? NULL : fopen(inName2, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", inName2, strerror(errno)), exit(1); errno = 0; FILE *otFile1 = (otName1 == NULL) ? NULL : fopen(otName1, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", otName1, strerror(errno)), exit(1); errno = 0; FILE *otFile2 = (otName2 == NULL) ? NULL : fopen(otName2, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", otName2, strerror(errno)), exit(1); // Load reads. vector reads; pairedRead pr; while (!feof(inFile1)) { uint32 seq1=0, bgn1=0, end1=0; uint32 seq2=0, bgn2=0, end2=0; char *a = readRead(inFile1, seq1, bgn1, end1); char *b = readRead(inFile2, seq2, bgn2, end2); if (feof(inFile1)) break; if ((a != NULL) && (b != NULL)) { pr.seqId = (seq1 < seq2) ? seq1 : seq2; pr.seqPos = (bgn1 < end1) ? bgn1 : end1; assert(bgn1 == bgn2); assert(end1 == end2); } else if (a != NULL) { pr.seqId = seq1; pr.seqPos = bgn1; } else { assert(0); } pr.readA = a; pr.readB = b; reads.push_back(pr); } fprintf(stderr, "Loaded "F_U64" mated reads.\n", reads.size()); sort(reads.begin(), reads.end()); for (uint32 i=0; i #include #include #include #include "AS_global.H" #include "AS_UTL_fileIO.H" #include "AS_UTL_reverseComplement.H" #include using namespace std; #undef DEBUG_ERRORS // Print when mismatch, insert or delete errors are added vector seqStartPositions; static char revComp[256]; static char errorBase[256][3]; static char insertBase[4]; static char validBase[256]; double readMismatchRate = 0.01; // Fraction mismatch error double readInsertRate = 0.01; // Fraction insertion error double readDeleteRate = 0.01; // Fraction deletion error bool allowGaps = false; bool allowNs = false; const uint32 mpJunctionsNone = 0; const uint32 mpJunctionsNormal = 1; const uint32 mpJunctionsAlways = 2; double pRevComp = 0.5; double pNormal = 0.0; #define QV_BASE '!' // Returns random int in range bgn <= x < end. // int32 randomUniform(int32 bgn, int32 end) { if (bgn >= end) fprintf(stderr, "randomUniform()-- ERROR: invalid range bgn=%d end=%d\n", bgn, end); assert(bgn < end); return((int32)floor((end - bgn) * drand48() + bgn)); } // Generate a random gaussian using the Marsaglia polar method. // int32 randomGaussian(double mean, double stddev) { double u = 0.0; double v = 0.0; double r = 0.0; do { u = 2.0 * drand48() - 1.0; v = 2.0 * drand48() - 1.0; r = u * u + v * v; } while (r >= 1.0); if (r < 1e-10) r = 1e-10; r = sqrt(-2 * log(r) / r); // Uniform gaussian is u*r and v*r. We only use one of these. return((int32)(mean + u * r * stddev)); } int32 findSequenceIndex(int32 pos) { int32 seqIdx = 0; for (seqIdx=0; seqIdx 0); seqIdx--; return(seqIdx); } uint64 nNoChange = 0; uint64 nMismatch = 0; uint64 nInsert = 0; uint64 nDelete = 0; void makeSequenceError(char *s1, char *q1, int32 &p) { double r = drand48(); if ((r < readMismatchRate) && (p >= 0)) { #ifdef DEBUG_ERRORS fprintf(stderr, "MISMATCH at p=%d base=%d/%c qc=%d/%c (INITIAL)\n", p, s1[p], s1[p], q1[p], q1[p]); #endif s1[p] = errorBase[s1[p]][randomUniform(0, 3)]; q1[p] = (validBase[s1[p]]) ? QV_BASE + 8 : QV_BASE + 2; nMismatch++; #ifdef DEBUG_ERRORS fprintf(stderr, "MISMATCH at p=%d base=%d/%c qc=%d/%c\n", p, s1[p], s1[p], q1[p], q1[p]); #endif return; } r -= readMismatchRate; if (r < readInsertRate) { p++; s1[p] = insertBase[randomUniform(0, 4)]; q1[p] = (validBase[s1[p]]) ? QV_BASE + 4 : QV_BASE + 2; nInsert++; #ifdef DEBUG_ERRORS fprintf(stderr, "INSERT at p=%d base=%d/%c qc=%d/%c\n", p, s1[p], s1[p], q1[p], q1[p]); #endif return; } r -= readInsertRate; if ((r < readDeleteRate) && (p > 0)) { p--; nDelete++; #ifdef DEBUG_ERRORS fprintf(stderr, "DELETE at p=%d\n", p); #endif return; } r -= readDeleteRate; nNoChange++; } bool makeSequences(char *frag, int32 fragLen, int32 readLen, char *s1, char *q1, char *s2, char *q2, bool makeNormal = false) { for (int32 p=0, i=0; p') || ((allowGaps == false) && (seq[i] == 'N'))) goto trySEagain; // Generate the sequence. if (makeSequences(seq + bgn, 0, readLen, s1, q1, NULL, NULL) == false) goto trySEagain; // Make sure the read doesn't contain N's (redundant in this particular case) if (allowNs == false) for (int32 i=0; i') || ((allowGaps == false) && (seq[i] == 'N'))) goto tryPEagain; // Read sequences from the ends. bool makeNormal = ((pNormal > 0.0) && (drand48() < pNormal)); if (makeSequences(seq + bgn, len, readLen, s1, q1, s2, q2, makeNormal) == false) goto tryPEagain; // Make sure the reads don't contain N's if (allowNs == false) for (int32 i=0; i') || ((allowGaps == false) && (seq[i] == 'N'))) goto tryMPagain; // If we fail the mpEnrichment test, pick a random shearing and return PE reads. // Otherwise, rotate the sequence to circularize and return MP reads. if (mpEnrichment < drand48()) { // Failed to wash away non-biotin marked sequence, make PE int32 sbgn = bgn + randomUniform(0, len - slen); bool makeNormal = ((pNormal > 0.0) && (drand48() < pNormal)); if (makeSequences(seq + sbgn, slen, readLen, s1, q1, s2, q2, makeNormal) == false) goto tryMPagain; // Make sure the reads don't contain N's if (allowNs == false) for (int32 i=0; i= slen)) fprintf(stderr, "ERROR: invalid shift %d.\n", shift); assert(shift > 0); assert(shift < slen); // Put 'shift' bases from the end of the insert on the start of sh[], // and then fill the remaining of sh[] with the beginning of the insert. // // sh[] == [------>END] [BGN--------->] int32 pInsert = bgn; int32 pShift = shift; while (pShift < slen) // Copy BGN---> sh[pShift++] = seq[pInsert++]; pInsert = bgn + len - shift; pShift = 0; while (pInsert < bgn + len) // Copy --->END sh[pShift++] = seq[pInsert++]; assert(pShift == shift); sh[slen] = 0; bool makeNormal = ((pNormal > 0.0) && (drand48() < pNormal)); if (makeSequences(sh, slen, readLen, s1, q1, s2, q2, makeNormal) == false) goto tryMPagain; // Make sure the reads don't contain N's if (allowNs == false) for (int32 i=0; i= slen - readLen) type = (type == 'a') ? 'c' : 'b'; if (mpJunctions == mpJunctionsNone) assert(type == 't'); if (mpJunctions == mpJunctionsAlways) assert(type != 't'); // Add a marker for the chimeric point. This unfortunately includes some knowledge of // makeSequences(); the second sequence is reverse complemented. In that case, adjust shift // to the the position in that reverse complemented read. // if ((shift > 0) && (shift < readLen)) { q1[shift-1] = QV_BASE + 10; q1[shift-0] = QV_BASE + 10; } if ((shift > slen - readLen) && (shift < slen)) { assert((readLen - (shift + readLen - slen)) > 0); assert((readLen - (shift + readLen - slen)) < readLen); shift = readLen - (shift + readLen - slen); q2[shift - 1] = QV_BASE + 10; q2[shift - 0] = QV_BASE + 10; } // Output sequences, with a descriptive ID. Because bowtie2 removes /1 and /2 when the // mate maps concordantly, we no longer use that form. fprintf(outputI, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#1\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift); fprintf(outputI, "%s\n", s1); fprintf(outputI, "+\n"); fprintf(outputI, "%s\n", q1); fprintf(outputI, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#2\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift); fprintf(outputI, "%s\n", s2); fprintf(outputI, "+\n"); fprintf(outputI, "%s\n", q2); fprintf(output1, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#1\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift); fprintf(output1, "%s\n", s1); fprintf(output1, "+\n"); fprintf(output1, "%s\n", q1); fprintf(output2, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#2\n", type, (makeNormal) ? "normal" : "", np, idx, bgn, bgn+len, shift, slen, bgn+len-shift); fprintf(output2, "%s\n", s2); fprintf(output2, "+\n"); fprintf(output2, "%s\n", q2); reverseComplement(s1, q1, readLen); reverseComplement(s2, q2, readLen); fprintf(outputC, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#1\n", type, (makeNormal) ? "normal" : "", np, idx, bgn+len, bgn, shift, slen, bgn+len-shift); fprintf(outputC, "%s\n", s1); fprintf(outputC, "+\n"); fprintf(outputC, "%s\n", q1); fprintf(outputC, "@%cMP%s_%d_%d@%d-%d_%d/%d/%d#2\n", type, (makeNormal) ? "normal" : "", np, idx, bgn+len, bgn, shift, slen, bgn+len-shift); fprintf(outputC, "%s\n", s2); fprintf(outputC, "+\n"); fprintf(outputC, "%s\n", q2); } //if ((np % 1000) == 0) // fprintf(stderr, "%9d / %9d - %5.2f%%\r", np, numPairs, 100.0 * np / numPairs); } } void makeCC(char *seq, int32 seqLen, FILE *outputI, FILE *outputC, int32 readLen, int32 numReads, int32 ccJunkSize, int32 ccJunkStdDev, double ccFalse) { char acgt[4] = { 'A', 'C', 'G', 'T' }; char *s1 = new char [readLen + 1]; char *q1 = new char [readLen + 1]; for (int32 nr=0; nr readLen - 80) goto tryCCagain; int32 lenf = randomUniform(1, readLen - lenj); int32 lenr = readLen - lenj - lenf; if ((lenf < 1) || (lenr < 1)) goto tryCCagain; int32 bgnf = randomUniform(1, seqLen - readLen); int32 idxf = findSequenceIndex(bgnf); int32 zerf = seqStartPositions[idxf]; int32 bgnr = randomUniform(1, seqLen - readLen); int32 idxr = findSequenceIndex(bgnr); int32 zerr = seqStartPositions[idxr]; bool isFalse = false; if (ccFalse < drand48()) { bgnr = bgnf + readLen - lenr; idxr = findSequenceIndex(bgnr); zerr = seqStartPositions[idxr]; isFalse = true; } // Scan the sequences, if we spanned a sequence break or encounter a block of Ns, don't use this pair if (idxf != idxr) goto tryCCagain; if (allowNs == false) for (int32 i=bgnf; i') || ((allowGaps == false) && (seq[i] == 'N'))) goto tryCCagain; if (allowNs == false) for (int32 i=bgnr; i') || ((allowGaps == false) && (seq[i] == 'N'))) goto tryCCagain; // Generate the sequence. if ((makeSequences(seq + bgnf, 0, lenf, s1, q1, NULL, NULL) == false) || (makeSequences(seq + bgnr, 0, lenr, s1 + readLen - lenr, q1 + readLen - lenr, NULL, NULL) == false)) goto tryCCagain; // Load the read with random garbage. for (int32 i=lenf; i= argc) { fprintf(stderr, "Not enough args to -pe.\n"); err++; } else { peEnable = true; peShearSize = atoi(argv[++arg]); peShearStdDev = atoi(argv[++arg]); } } else if (strcmp(argv[arg], "-mp") == 0) { if (arg + 5 >= argc) { fprintf(stderr, "Not enough args to -mp.\n"); err++; } else { mpEnable = true; mpInsertSize = atoi(argv[++arg]); mpInsertStdDev = atoi(argv[++arg]); mpShearSize = atoi(argv[++arg]); mpShearStdDev = atoi(argv[++arg]); mpEnrichment = atof(argv[++arg]); } } else if (strcmp(argv[arg], "-cc") == 0) { if (arg + 3 >= argc) { fprintf(stderr, "Not enough args to -cc.\n"); err++; } else { ccEnable = true; ccJunkSize = atoi(argv[++arg]); ccJunkStdDev = atoi(argv[++arg]); ccFalse = atof(argv[++arg]); } } else if (strcmp(argv[arg], "-seed") == 0) { seed = atoi(argv[++arg]); } else { fprintf(stderr, "Unknown arg '%s'\n", argv[arg]); err++; } arg++; } if ((err) || (fastaName == NULL) || (outputPrefix == NULL) || ((seEnable == false) && (peEnable == false) && (mpEnable == false) && (ccEnable == false)) || ((seEnable == true) && (cloneCoverage > 0))) { fprintf(stderr, "usage: %s -f reference.fasta -o output-prefix -l read-length ....\n", argv[0]); fprintf(stderr, " -f ref.fasta Use sequences in ref.fasta as the genome.\n"); fprintf(stderr, " -o name Create outputs name.1.fastq and name.2.fastq (and maybe others).\n"); fprintf(stderr, " -l len Create reads of length 'len' bases.\n"); fprintf(stderr, " -n n Create 'n' reads (for -se) or 'n' pairs of reads (for -pe and -mp).\n"); fprintf(stderr, " -x read-cov Set 'np' to create reads that sample the genome to 'read-cov' read coverage.\n"); fprintf(stderr, " -X clone-cov Set 'np' to create reads that sample the genome to 'clone-cov' clone coverage.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -em err Reads will contain fraction mismatch error 'e' (0.01 == 1%% error).\n"); fprintf(stderr, " -ei err Reads will contain fraction insertion error 'e' (0.01 == 1%% error).\n"); fprintf(stderr, " -ed err Reads will contain fraction deletion error 'e' (0.01 == 1%% error).\n"); fprintf(stderr, "\n"); fprintf(stderr, " -seed s Seed randomness with 32-bit integer s.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -allowgaps Allow pairs to span N regions in the reference. By default, pairs\n"); fprintf(stderr, " are not allowed to span a gap. Reads are never allowed to cover N's.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -allowns Allow reads to contain N regions. Implies -allowgaps\n"); fprintf(stderr, "\n"); fprintf(stderr, " -nojunction For -mp, do not create chimeric junction reads. Create only fully PE or\n"); fprintf(stderr, " fully MP reads.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -normal p Output a normal-oriented (both forward or both reverse) pair with\n"); fprintf(stderr, " probability p. Only for -pe and -mp.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -se\n"); fprintf(stderr, " Create single-end reads.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -cc junkSize junkStdDev false\n"); fprintf(stderr, " Create chimeric single-end reads. The chimer is formed from two uniformly\n"); fprintf(stderr, " distributed positions in the reference. Some amount of random junk is inserted\n"); fprintf(stderr, " at the junction. With probability 'false' the read is not chimeric, but still\n"); fprintf(stderr, " the junk bases inserted in the middle.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -pe shearSize shearStdDev\n"); fprintf(stderr, " Create paired-end reads, from fragments of size 'shearSize +- shearStdDev'.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -mp insertSize insertStdDev shearSize shearStdDev enrichment\n"); fprintf(stderr, " Create mate-pair reads. The pairs will be 'insertSize +- insertStdDev'\n"); fprintf(stderr, " apart. The circularized insert is then sheared into fragments of size\n"); fprintf(stderr, " 'shearSize +- shearStdDev'. With probability 'enrichment' the fragment\n"); fprintf(stderr, " containing the junction is used to form the pair of reads. The junction\n"); fprintf(stderr, " location is uniformly distributed through this fragment.\n"); fprintf(stderr, " Reads are labeled as:\n"); fprintf(stderr, " tMP - a MP pair\n"); fprintf(stderr, " fMP - a PE pair\n"); fprintf(stderr, " aMP - a MP pair with junction in the first read\n"); fprintf(stderr, " bMP - a MP pair with junction in the second read\n"); fprintf(stderr, " cMP - a MP pair with junction in both reads (the reads overlap)\n"); fprintf(stderr, "\n"); fprintf(stderr, "Output QV's are the Sanger spec.\n"); fprintf(stderr, "\n"); if (fastaName == NULL) fprintf(stderr, "ERROR: No fasta file (-f) supplied.\n"); if (outputPrefix == NULL) fprintf(stderr, "ERROR: No output prefix (-o) supplied.\n"); if ((seEnable == false) && (peEnable == false) && (mpEnable == false) && (ccEnable == false)) fprintf(stderr, "ERROR: No type (-se or -pe or -mp) selected.\n"); if ((seEnable == true) && (cloneCoverage > 0)) fprintf(stderr, "ERROR: Can't sample clone coverage with single-ended (-se) reads.\n"); exit(1); } if ((readMismatchRate < 0.0) || (readMismatchRate > 1.0)) err++; if ((readInsertRate < 0.0) || (readInsertRate > 1.0)) err++; if ((readDeleteRate < 0.0) || (readDeleteRate > 1.0)) err++; if (readMismatchRate + readInsertRate + readDeleteRate > 1.0) err++; if (err > 0) { fprintf(stderr, "Invalid error rates.\n"); exit(1); } // // Initialize // // The errorBase[0] assignments permit makeSequenceError() to return a result // when the end of genome is hit. This is caught later in makeSequence() and the // read is aborted. fprintf(stderr, "seed = "F_U64"\n", seed); srand48(seed); memset(revComp, '&', sizeof(char) * 256); revComp['A'] = 'T'; revComp['C'] = 'G'; revComp['G'] = 'C'; revComp['T'] = 'A'; revComp['N'] = 'N'; memset(errorBase, '*', sizeof(char) * 256 * 3); errorBase[ 0 ][0] = 0 ; errorBase[ 0 ][1] = 0 ; errorBase[ 0 ][2] = 0 ; errorBase['A'][0] = 'C'; errorBase['A'][1] = 'G'; errorBase['A'][2] = 'T'; errorBase['C'][0] = 'A'; errorBase['C'][1] = 'G'; errorBase['C'][2] = 'T'; errorBase['G'][0] = 'A'; errorBase['G'][1] = 'C'; errorBase['G'][2] = 'T'; errorBase['T'][0] = 'A'; errorBase['T'][1] = 'C'; errorBase['T'][2] = 'G'; errorBase['N'][0] = 'N'; errorBase['N'][1] = 'N'; errorBase['N'][2] = 'N'; memset(insertBase, '*', sizeof(char) * 4); insertBase[0] = 'A'; insertBase[1] = 'C'; insertBase[2] = 'G'; insertBase[3] = 'T'; memset(validBase, 0, sizeof(char) * 256); validBase['A'] = 1; validBase['C'] = 1; validBase['G'] = 1; validBase['T'] = 1; // // Open output files, failing quickly. // errno = 0; if ((seEnable == true) || (ccEnable == true)) { sprintf(outputName, "%s.s.fastq", outputPrefix); outputI = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open output file '%s': %s\n", outputName, strerror(errno)), exit(1); } if ((seEnable == false) && (ccEnable == false)) { sprintf(outputName, "%s.i.fastq", outputPrefix); outputI = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open output file '%s': %s\n", outputName, strerror(errno)), exit(1); sprintf(outputName, "%s.c.fastq", outputPrefix); outputC = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open output file '%s': %s\n", outputName, strerror(errno)), exit(1); } if (peEnable || mpEnable) { sprintf(outputName, "%s.1.fastq", outputPrefix); output1 = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open output file '%s': %s\n", outputName, strerror(errno)), exit(1); sprintf(outputName, "%s.2.fastq", outputPrefix); output2 = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open output file '%s': %s\n", outputName, strerror(errno)), exit(1); } // // Load all reference sequences into a single string. Seperate different sequences with a '>', we'll not make // fragments that span these markers (inefficiently, sigh). // fastaFile = fopen(fastaName, "r"); if (errno) fprintf(stderr, "Failed to open fasta file '%s': %s\n", fastaName, strerror(errno)), exit(1); numSeq = 0; seqMax = AS_UTL_sizeOfFile(fastaName); seqLen = 0; seq = new char [seqMax + 1]; memset(seq, 0, sizeof(char) * seqMax); uint32 nInvalid = 0; while (!feof(fastaFile)) { fgets(seq + seqLen, seqMax - seqLen, fastaFile); if (seq[seqLen] == '>') { numSeq++; seqLen++; seqStartPositions.push_back(seqLen); continue; } for (; ((seq[seqLen] != '\n') && (seq[seqLen] != '\r') && (seq[seqLen] != 0)); seqLen++) { seq[seqLen] = toupper(seq[seqLen]); if ((seq[seqLen] != 'N') && (validBase[seq[seqLen]] == 0)) { nInvalid++; //fprintf(stderr, "Replace invalid base '%c' at position %u.\n", seq[seqLen], seqLen); seq[seqLen] = insertBase[randomUniform(0, 3)]; //q1[p] = (validBase[s1[p]]) ? QV_BASE + 8 : QV_BASE + 2; } } assert(seqLen < seqMax); } fclose(fastaFile); seq[seqLen] = 0; assert(numSeq == seqStartPositions.size()); fprintf(stderr, "Loaded %u sequences of length %d, with %u invalid bases fixed.\n", numSeq, seqLen - numSeq, nInvalid); // // If requested, compute the number of pairs to get a desired X of coverage // { uint32 cloneSize = 0; uint32 cloneStdDev = 0; uint32 readNumReads = UINT32_MAX; uint32 readNumPairs = UINT32_MAX; uint32 cloneNumReads = UINT32_MAX; uint32 cloneNumPairs = UINT32_MAX; if (peEnable) { cloneSize = peShearSize; cloneStdDev = peShearStdDev; } if (mpEnable) { cloneSize = mpInsertSize; cloneStdDev = mpInsertStdDev; } if (ccEnable) { cloneSize = ccJunkSize; cloneStdDev = ccJunkStdDev; } if (readCoverage > 0) { readNumReads = (uint32)floor(readCoverage * (seqLen - numSeq) / readLen); readNumPairs = readNumReads / 2; } if ((cloneCoverage > 0) && (seEnable == false)) { cloneNumPairs = (uint32)floor(cloneCoverage * (seqLen - numSeq) / cloneSize); cloneNumReads = cloneNumPairs * 2; } numReads = MIN(numReads, readNumReads); numPairs = MIN(numPairs, readNumPairs); numReads = MIN(numReads, cloneNumReads); numPairs = MIN(numPairs, cloneNumPairs); if (seEnable) fprintf(stderr, "Generate %.2f X read coverage of a %dbp genome with %u %dbp reads.\n", (double)numReads * readLen / seqLen, seqLen - numSeq, numReads, readLen); else fprintf(stderr, "Generate %.2f X read (%.2f X clone) coverage of a %dbp genome with %u pairs of %dbp reads from a clone of %d +- %dbp.\n", (double)numReads * readLen / seqLen, (double)numPairs * cloneSize / seqLen, seqLen - numSeq, numPairs, readLen, cloneSize, cloneStdDev); } // // // if (seEnable) makeSE(seq, seqLen, outputI, outputC, readLen, numReads); if (peEnable) makePE(seq, seqLen, outputI, outputC, output1, output2, readLen, numPairs, peShearSize, peShearStdDev); if (mpEnable) makeMP(seq, seqLen, outputI, outputC, output1, output2, readLen, numPairs, mpInsertSize, mpInsertStdDev, mpShearSize, mpShearStdDev, mpEnrichment, mpJunctions); if (ccEnable) makeCC(seq, seqLen, outputI, outputC, readLen, numReads, ccJunkSize, ccJunkStdDev, ccFalse); // // // if ((seEnable == true) || (ccEnable == true)) fclose(outputI); if ((seEnable == false) && (ccEnable == false)) { fclose(outputI); fclose(outputC); } if (peEnable || mpEnable) { fclose(output1); fclose(output2); } delete [] seq; fprintf(stderr, "\n"); fprintf(stderr, "Number of reads with:\n"); fprintf(stderr, " nNoChange = "F_U64"\n", nNoChange); fprintf(stderr, " nMismatch = "F_U64"\n", nMismatch); fprintf(stderr, " nInsert = "F_U64"\n", nInsert); fprintf(stderr, " nDelete = "F_U64"\n", nDelete); exit(0); } canu-1.3/src/fastq-utilities/fastqSimulate.mk000066400000000000000000000007331272335422300214020ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := fastqSimulate SOURCES := fastqSimulate.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/main.mk000066400000000000000000000151351272335422300143570ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${DESTDIR})" "" DESTDIR := endif ifeq "$(strip ${PREFIX})" "" ifeq "$(strip ${DESTDIR})" "" PREFIX := $(realpath ..) else PREFIX := /canu endif endif ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := $(DESTDIR)$(PREFIX)/$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := $(DESTDIR)$(PREFIX)/$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := libcanu.a SOURCES := AS_global.C \ \ AS_UTL/AS_UTL_decodeRange.C \ AS_UTL/AS_UTL_fasta.C \ AS_UTL/AS_UTL_fileIO.C \ AS_UTL/AS_UTL_reverseComplement.C \ AS_UTL/AS_UTL_stackTrace.C \ \ AS_UTL/AS_UTL_alloc.C \ \ AS_UTL/bitEncodings.C \ AS_UTL/bitPackedFile.C \ AS_UTL/bitPackedArray.C \ AS_UTL/dnaAlphabets.C \ AS_UTL/md5.C \ AS_UTL/mt19937ar.C \ AS_UTL/readBuffer.C \ AS_UTL/speedCounter.C \ AS_UTL/sweatShop.C \ AS_UTL/timeAndSize.C \ AS_UTL/kMer.C \ \ stores/gkStore.C \ stores/gkStoreEncode.C \ \ stores/ovOverlap.C \ stores/ovStore.C \ stores/ovStoreFile.C \ \ stores/tgStore.C \ stores/tgTig.C \ stores/tgTigSizeAnalysis.C \ stores/tgTigMultiAlignDisplay.C \ \ meryl/libmeryl.C \ \ overlapInCore/overlapReadCache.C \ \ overlapErrorAdjustment/analyzeAlignment.C \ \ overlapInCore/liboverlap/Binomial_Bound.C \ overlapInCore/liboverlap/Display_Alignment.C \ overlapInCore/liboverlap/prefixEditDistance.C \ overlapInCore/liboverlap/prefixEditDistance-allocateMoreSpace.C \ overlapInCore/liboverlap/prefixEditDistance-extend.C \ overlapInCore/liboverlap/prefixEditDistance-forward.C \ overlapInCore/liboverlap/prefixEditDistance-reverse.C \ \ utgcns/libNDalign/NDalign.C \ \ utgcns/libNDalign/Binomial_Bound.C \ utgcns/libNDalign/NDalgorithm.C \ utgcns/libNDalign/NDalgorithm-allocateMoreSpace.C \ utgcns/libNDalign/NDalgorithm-extend.C \ utgcns/libNDalign/NDalgorithm-forward.C \ utgcns/libNDalign/NDalgorithm-reverse.C \ \ utgcns/libcns/abAbacus-addRead.C \ utgcns/libcns/abAbacus-appendBases.C \ utgcns/libcns/abAbacus-applyAlignment.C \ utgcns/libcns/abAbacus-baseCall.C \ utgcns/libcns/abAbacus-mergeRefine.C \ utgcns/libcns/abAbacus-refine.C \ utgcns/libcns/abAbacus-refreshMultiAlign.C \ utgcns/libcns/abAbacus.C \ utgcns/libcns/abColumn.C \ utgcns/libcns/abMultiAlign.C \ utgcns/libcns/unitigConsensus.C \ utgcns/libpbutgcns/Alignment.C \ utgcns/libpbutgcns/AlnGraphBoost.C \ utgcns/libpbutgcns/SimpleAligner.C \ utgcns/libNDFalcon/dw.C \ \ falcon_sense/libfalcon/falcon.C \ falcon_sense/libfalcon/kmer_lookup.C SRC_INCDIRS := . \ AS_UTL \ stores \ alignment \ utgcns/libNDalign \ utgcns/libcns \ utgcns/libpbutgcns \ utgcns/libNDFalcon \ utgcns/libboost \ meryl/libleaff \ overlapInCore \ overlapInCore/liboverlap \ falcon_sense/libfalcon SUBMAKEFILES := stores/gatekeeperCreate.mk \ stores/gatekeeperDumpFASTQ.mk \ stores/gatekeeperDumpMetaData.mk \ stores/gatekeeperPartition.mk \ stores/ovStoreBuild.mk \ stores/ovStoreBucketizer.mk \ stores/ovStoreSorter.mk \ stores/ovStoreIndexer.mk \ stores/ovStoreDump.mk \ stores/ovStoreStats.mk \ stores/tgStoreCompress.mk \ stores/tgStoreDump.mk \ stores/tgStoreLoad.mk \ stores/tgStoreFilter.mk \ stores/tgStoreCoverageStat.mk \ stores/tgTigDisplay.mk \ \ meryl/libleaff.mk \ meryl/leaff.mk \ meryl/meryl.mk \ meryl/simple.mk \ meryl/estimate-mer-threshold.mk \ \ overlapInCore/overlapInCore.mk \ overlapInCore/overlapInCorePartition.mk \ overlapInCore/overlapConvert.mk \ overlapInCore/overlapImport.mk \ overlapInCore/overlapPair.mk \ \ overlapInCore/liboverlap/prefixEditDistance-matchLimitGenerate.mk \ \ mhap/mhap.mk \ mhap/mhapConvert.mk \ \ minimap/mmapConvert.mk \ \ correction/filterCorrectionOverlaps.mk \ correction/generateCorrectionLayouts.mk \ correction/readConsensus.mk \ correction/errorEstimate.mk \ \ falcon_sense/createFalconSenseInputs.mk \ falcon_sense/falcon_sense.mk \ \ overlapBasedTrimming/trimReads.mk \ overlapBasedTrimming/splitReads.mk \ \ overlapErrorAdjustment/findErrors.mk \ overlapErrorAdjustment/findErrors-Dump.mk \ overlapErrorAdjustment/correctOverlaps.mk \ \ bogart/bogart.mk \ bogart/buildGraph.mk \ \ bogus/bogus.mk \ \ erateEstimate/erateEstimate.mk \ \ utgcns/utgcns.mk \ \ fastq-utilities/fastqAnalyze.mk \ fastq-utilities/fastqSample.mk \ fastq-utilities/fastqSimulate.mk \ fastq-utilities/fastqSimulate-sort.mk canu-1.3/src/merTrim/000077500000000000000000000000001272335422300145145ustar00rootroot00000000000000canu-1.3/src/merTrim/merTrim-compare-logs.pl000066400000000000000000000060001272335422300210520ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/AS_MER/merTrim-compare-logs.pl # # Modifications by: # # Brian P. Walenz from 2014-NOV-15 to 2014-DEC-05 # are Copyright 2014 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $log1 = shift @ARGV; my $log2 = shift @ARGV; if (!defined($log1) || !defined($log2)) { die "usage: $0 run1.log run2.log\n"; } open(L1, "< $log1") or die; open(L2, "< $log1") or die; my ($a1, $a2, $a3, $a4, $a5, $a6, $a7, $a8, $a9); my ($b1, $b2, $b3, $b4, $b5, $b6, $b7, $b8, $b9); while (!eof(L1) && !eof(L2)) { anotherA: do { $a1 = ; # FINAL, or "Correct" lines } while ($a1 !~ m/^FINAL/); $a2 = ; # ORI seq $a3 = ; # COR seq $a4 = ; # COR qlt $a5 = ; # COVERAGE $a6 = ; # CORRECTIONS $a7 = ; # DISCONNECTION $a8 = ; # ADAPTER $a9 = ; # RESULT #if ($a1 =~ m/^ADAPTERSEARCH/) { # goto anotherA; #} anotherB: do { $b1 = ; # FINAL, or "Correct" lines } while ($b1 !~ m/^FINAL/); $b2 = ; # ORI seq $b3 = ; # COR seq $b4 = ; # COR qlt $b5 = ; # COVERAGE $b6 = ; # CORRECTIONS $b7 = ; # DISCONNECTION $b8 = ; # ADAPTER $b9 = ; # RESULT #if ($b1 =~ m/^ADAPTERSEARCH/) { # goto anotherB; #} my ($aID, $aLen, $aBgn, $aEnd); my ($bID, $bLen, $bBgn, $bEnd); # FINAL or ADAPTERSEARCH if ($a1 =~ m/^\w+\sread\s(\d+)\slen\s(\d+)\s\(trim\s(\d+)-(\d+)\)$/) { $aID = $1; $aLen = $2; $aBgn = $3; $aEnd = $4; } else { die "Nope a1 $a1"; } if ($b1 =~ m/^\w+\sread\s(\d+)\slen\s(\d+)\s\(trim\s(\d+)-(\d+)\)$/) { $bID = $1; $bLen = $2; $bBgn = $3; $bEnd = $4; } else { die "Nope b1 $b1"; } die "ID mismatch $aID $bID\n" if ($aID != $bID); if (($aBgn != $bBgn) || ($aEnd != $bEnd)) { print "$aID/$bID $aLen/$bLen $aBgn-$aEnd $bBgn-$bEnd\n"; } if (($aID % 10000) == 0) { print STDERR "$aID\n"; } } canu-1.3/src/merTrim/merTrim.C000066400000000000000000001772451272335422300162570ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_MER/merTrim.C * * Modifications by: * * Brian P. Walenz from 2010-FEB-22 to 2014-APR-11 * are Copyright 2010-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2010-JUL-12 * are Copyright 2010 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-23 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "AS_UTL_reverseComplement.H" #include "AS_PER_gkpStore.H" #include "AS_PER_encodeSequenceQuality.H" // QUALITY_MAX, QV conversion #include "AS_OVS_overlapStore.H" #include #include "AS_MER_gkpStore_to_FastABase.H" #include "bio++.H" #include "sweatShop.H" #include "existDB.H" #include "positionDB.H" #include "libmeryl.H" #include "logMsg.H" #include "merTrimResult.H" // There is a serious bug in storing the kmer count in the existDB that merTrim 1.40 exposes. kmer // must be at least r1950 to fix the bug. #if !defined(EXISTDB_H_VERSION) || (EXISTDB_H_VERSION < 1960) #error kmer needs to be updated to at least r1960 #error note that the kmer svn url changed in mid December 2012, the old url does not have r1960 #endif uint32 VERBOSE = 0; #define ALLGOOD 1 #define ALLCRAP 2 #define ATTEMPTCORRECTION 3 // Doesn't work, gets different results. #undef USE_MERSTREAM_REBUILD #undef TEST_TESTBASE char *createAdapterString(bool adapIllumina, bool adap454); class mertrimGlobalData { public: mertrimGlobalData() { gkpPath = 0L; fqInputPath = 0L; fqOutputPath = 0L; fqVerifyPath = 0L; merSize = 22; merCountsFile = 0L; merCountsCache = false; adapCountsFile = 0L; adapIllumina = false; adap454 = false; compression = 0; // Does not work! numThreads = 4; beVerbose = false; forceCorrection = false; correctMismatch = true; correctIndel = true; actualCoverage = 0; // Estimate of coverage minCorrectFraction = 1.0 / 3.0; // Base can be corrected if less than 1/3 coverage minCorrect = 0; // minVerifiedFraction = 1.0 / 4.0; // Base can be corrected to something with only 1/4 coverage minVerified = 0; // endTrimDefault = true; endTrimNum = 0; #if 0 // These never worked well. endTrimWinScale[0] = 0.50; endTrimErrAllow[0] = 2; endTrimWinScale[1] = 0.25; endTrimErrAllow[1] = 0; #endif endTrimQV = '2'; doTrimming = true; discardZeroCoverage = false; discardImperfectCoverage = false; trimImperfectCoverage = true; gkRead = NULL; fqInput = NULL; fqOutput = NULL; fqVerify = NULL; fqLog = NULL; genomicDB = NULL; adapterDB = NULL; resPath = NULL; resFile = NULL; gktBgn = 0; gktEnd = 0; gktCur = 0; }; ~mertrimGlobalData() { delete gkRead; delete fqInput; delete fqOutput; delete fqVerify; delete fqLog; delete genomicDB; delete adapterDB; if (resFile != NULL) fclose(resFile); }; void initializeGatekeeper(void) { if (gkpPath == NULL) return; fprintf(stderr, "opening gkStore '%s'\n", gkpPath); gkRead = gkStore::gkStore_open(gkpPath, FALSE, FALSE); if (gktBgn == 0) { gktBgn = 1; gktEnd = gkRead->gkStore_getNumFragments(); } gktCur = gktBgn; if (gktBgn > gktEnd) fprintf(stderr, "ERROR: invalid range: -b ("F_U32") >= -e ("F_U32").\n", gktBgn, gktEnd), exit(1); if (gktEnd > gkRead->gkStore_getNumFragments()) fprintf(stderr, "ERROR: invalid range: -e ("F_U32") > num frags ("F_U32").\n", gktEnd, gkRead->gkStore_getNumFragments()), exit(1); errno = 0; resFile = fopen(resPath, "w"); if (errno) fprintf(stderr, "Failed to open output file '%s': %s\n", resPath, strerror(errno)), exit(1); }; void initializeFASTQ(void) { if (fqInputPath == NULL) return; if (fqOutputPath == NULL) return; fqInput = new compressedFileReader(fqInputPath); fqOutput = new compressedFileWriter(fqOutputPath); if (fqVerifyPath) fqVerify = new compressedFileReader(fqVerifyPath); char fqName[FILENAME_MAX]; sprintf(fqName, "%s.log", fqOutputPath); fqLog = new compressedFileWriter(fqName); }; void initialize(void) { initializeGatekeeper(); initializeFASTQ(); if (actualCoverage == 0) { merylStreamReader *MF = new merylStreamReader(merCountsFile); uint32 i = 0; uint32 iX = 0; //fprintf(stderr, "distinct: "F_U64"\n", MF->numberOfDistinctMers()); //fprintf(stderr, "unique: "F_U64"\n", MF->numberOfUniqueMers()); //fprintf(stderr, "total: "F_U64"\n", MF->numberOfTotalMers()); //fprintf(stderr, "Xcoverage zero 1 0 "F_U64"\n", MF->histogram(1)); for (i=2; (i < MF->histogramLength()) && (MF->histogram(i-1) > MF->histogram(i)); i++) //fprintf(stderr, "Xcoverage drop "F_U32" "F_U64" "F_U64"\n", i, MF->histogram(i-1), MF->histogram(i)); ; iX = i - 1; for (; i < MF->histogramLength(); i++) { if (MF->histogram(iX) < MF->histogram(i)) { //fprintf(stderr, "Xcoverage incr "F_U32" "F_U64" "F_U64"\n", i, MF->histogram(iX), MF->histogram(i)); iX = i; } else { //fprintf(stderr, "Xcoverage drop "F_U32" "F_U64" "F_U64"\n", i, MF->histogram(iX), MF->histogram(i)); } } fprintf(stderr, "Guessed X coverage is "F_U32"\n", iX); delete MF; actualCoverage = iX; } if (minCorrectFraction > 0) minCorrect = (uint32)floor(minCorrectFraction * actualCoverage); if (minVerifiedFraction > 0) minVerified = (uint32)floor(minVerifiedFraction * actualCoverage); fprintf(stderr, "Use minCorrect="F_U32" minVerified="F_U32"\n", minCorrect, minVerified); if (minCorrect < minVerified) { fprintf(stderr, "WARNING!\n"); fprintf(stderr, "WARNING! minVerified (-verified) should be less than minCorrect (-correct).\n"); fprintf(stderr, "WARNING!\n"); } if (adapCountsFile) { fprintf(stderr, "loading adapter mer database.\n"); adapterDB = new existDB(adapCountsFile, merSize, existDBcounts, 0, UINT32_MAX); adapterDB->printState(stderr); } else if (adapIllumina || adap454) { fprintf(stderr, "creating adapter mer database.\n"); char *adapter = createAdapterString(adapIllumina, adap454); adapterDB = new existDB(adapter, merSize, existDBcanonical | existDBcounts); //adapterDB->printState(stderr); delete [] adapter; } else { fprintf(stderr, "not searching for adapter.\n"); } char cacheName[FILENAME_MAX]; sprintf(cacheName, "%s.merTrimDB", merCountsFile); if (AS_UTL_fileExists(cacheName, FALSE, FALSE)) { fprintf(stderr, "loading genome mer database from cache '%s'.\n", cacheName); genomicDB = new existDB(cacheName); } else if (merCountsFile) { fprintf(stderr, "loading genome mer database from meryl '%s'.\n", merCountsFile); genomicDB = new existDB(merCountsFile, merSize, existDBcounts, MIN(minCorrect, minVerified), UINT32_MAX); if (merCountsCache) { fprintf(stderr, "saving genome mer database to cache '%s'.\n", cacheName); genomicDB->saveState(cacheName); } } }; public: // Command line parameters // char *gkpPath; char *fqInputPath; char *fqOutputPath; char *fqVerifyPath; uint32 merSize; char *merCountsFile; bool merCountsCache; char *adapCountsFile; bool adapIllumina; bool adap454; uint32 compression; uint32 numThreads; bool beVerbose; bool forceCorrection; bool correctMismatch; bool correctIndel; char *resPath; FILE *resFile; bool endTrimDefault; uint32 endTrimNum; double endTrimWinScale[16]; uint32 endTrimErrAllow[16]; char endTrimQV; bool doTrimming; bool discardZeroCoverage; bool discardImperfectCoverage; bool trimImperfectCoverage; // Global data // gkStore *gkRead; uint32 actualCoverage; double minCorrectFraction; uint32 minCorrect; double minVerifiedFraction; uint32 minVerified; compressedFileReader *fqInput; compressedFileWriter *fqOutput; compressedFileReader *fqVerify; compressedFileWriter *fqLog; existDB *genomicDB; existDB *adapterDB; // Input State // uint32 gktBgn; uint32 gktCur; uint32 gktEnd; }; class mertrimThreadData { public: mertrimThreadData(mertrimGlobalData *g) { kb = new kMerBuilder(g->merSize, g->compression, 0L); }; ~mertrimThreadData() { delete kb; }; public: kMerBuilder *kb; }; class mertrimComputation { public: mertrimComputation() : log(false, 131072) { readName = NULL; origSeq = NULL; origQlt = NULL; corrSeq = NULL; corrQlt = NULL; seqMap = NULL; verifyName = NULL; verifySeq = NULL; verifyErr = NULL; rMS = NULL; disconnect = NULL; coverage = NULL; adapter = NULL; corrected = NULL; eDB = NULL; } ~mertrimComputation() { delete [] readName; delete [] origSeq; delete [] origQlt; delete [] corrSeq; delete [] corrQlt; delete [] verifyName; delete [] verifySeq; delete [] verifyErr; delete [] seqMap; delete rMS; delete [] disconnect; delete [] coverage; delete [] adapter; delete [] corrected; } void initializeGatekeeper(mertrimGlobalData *g_) { g = g_; readIID = fr.gkFragment_getReadIID(); seqLen = fr.gkFragment_getSequenceLength(); allocLen = seqLen + seqLen; readName = NULL; origSeq = new char [allocLen]; origQlt = new char [allocLen]; corrSeq = new char [allocLen]; corrQlt = new char [allocLen]; verifyName = NULL; verifySeq = NULL; verifyErr = NULL; seqMap = new uint32 [allocLen]; nMersExpected = 0; nMersTested = 0; nMersFound = 0; nMersCorrect = 0; rMS = NULL; disconnect = NULL; coverage = NULL; adapter = NULL; corrected = NULL; eDB = NULL; strcpy(origSeq, fr.gkFragment_getSequence()); strcpy(origQlt, fr.gkFragment_getQuality()); strcpy(corrSeq, fr.gkFragment_getSequence()); strcpy(corrQlt, fr.gkFragment_getQuality()); // Replace Ns with a random low-quality base. This is necessary, since the mer routines // will not make a mer for N, and we never see it to correct it. char letters[4] = { 'A', 'C', 'G', 'T' }; // Not really replacing N with random ACGT, but good enough for us. for (uint32 i=0; igktCur++; seqLen = 0; allocLen = AS_READ_MAX_NORMAL_LEN + AS_READ_MAX_NORMAL_LEN + 1; // Used for seq/qlt storage only allocLen = 65536; readName = new char [1024]; origSeq = new char [allocLen]; origQlt = new char [allocLen]; corrSeq = new char [allocLen]; corrQlt = new char [allocLen]; verifyName = NULL; // probably redundant verifySeq = NULL; verifyErr = NULL; seqMap = new uint32 [allocLen]; nMersExpected = 0; nMersTested = 0; nMersFound = 0; nMersCorrect = 0; rMS = NULL; disconnect = NULL; coverage = NULL; adapter = NULL; corrected = NULL; eDB = NULL; // Load the answer, if supplied (uses the real read storage space as temporary) if (g->fqVerify) { verifyName = new char [1024]; verifySeq = new char [allocLen]; verifyErr = new char [allocLen]; memset(verifyName, 1024, 0); memset(verifySeq, allocLen, 0); // Needed, for verified reads shorter than real reads memset(verifyErr, allocLen, '-'); fgets(verifyName, 1024, g->fqVerify->file()); fgets(verifySeq, allocLen, g->fqVerify->file()); fgets(origQlt, allocLen, g->fqVerify->file()); // qv name line, ignored fgets(origQlt, allocLen, g->fqVerify->file()); chomp(verifyName); chomp(verifySeq); } // Load a read to correct fgets(readName, 1024, g->fqInput->file()); fgets(origSeq, allocLen, g->fqInput->file()); fgets(origQlt, allocLen, g->fqInput->file()); // qv name line, ignored fgets(origQlt, allocLen, g->fqInput->file()); if (feof(g->fqInput->file())) return(false); chomp(readName); chomp(origSeq); chomp(origQlt); // Adjust QVs to CA encoding, bases to upper case, and non-acgt to acgt. #warning ASSUMING SANGER QV ENCODING for (uint32 i=0; origQlt[i]; i++) { if ('a' <= origSeq[i]) origSeq[i] += 'A' - 'a'; if (origQlt[i] < '!') fprintf(stderr, "ERROR: invalid QV '%c' (%d) in read '%s': '%s'\n", origQlt[i], origQlt[i], readName, origQlt); // Our Sanger reads (dumped as fastq from gkpStore) have QV's higher than this. //if ('J' < origQlt[i]) // fprintf(stderr, "ERROR: invalid QV '%c' (%d) in read '%s': '%s'\n", // origQlt[i], origQlt[i], readName, origQlt); origQlt[i] -= '!'; } // Copy to the corrected sequence strcpy(corrSeq, origSeq); strcpy(corrQlt, origQlt); // Replace Ns with a random low-quality base. This is necessary, since the mer routines // will not make a mer for N, and we never see it to correct it. uint32 numReplace = 0; char letters[4] = { 'A', 'C', 'G', 'T' }; // Not really replacing N with random ACGT, but good enough for us. for (uint32 i=0; corrSeq[i]; i++) if ((corrSeq[i] != 'A') && (corrSeq[i] != 'C') && (corrSeq[i] != 'G') && (corrSeq[i] != 'T')) { numReplace++; corrSeq[i] = letters[i & 0x03]; corrQlt[i] = 0; } seqLen = strlen(origSeq); allocLen = seqLen + seqLen; clrBgn = 0; clrEnd = seqLen; garbageInInput = false; gapInConfirmedKmers = false; hasNoConfirmedKmers = false; imperfectKmerCoverage = false; containsAdapter = false; containsAdapterCount = false; containsAdapterFixed = false; containsAdapterBgn = false; containsAdapterEnd = false; suspectedChimer = false; suspectedChimerBgn = 0; suspectedChimerEnd = 0; for (uint32 i=0; i= AS_OVL_ERROR_RATE * seqLen) { garbageInInput = true; clrBgn = 0; clrEnd = 0; } return(true); }; uint32 evaluate(void); void reverse(void); void analyze(void); uint32 testBases(char *bases, uint32 basesLen); uint32 testBaseChange(uint32 pos, char replacement); uint32 testBaseIndel(uint32 pos, char replacement); bool correctMismatch(uint32 pos, uint32 mNum, uint32 mExtra, bool isReversed); bool correctIndel(uint32 pos, uint32 mNum, uint32 mExtra, bool isReversed); void searchAdapter(bool isReversed); void scoreAdapter(void); void attemptCorrection(bool isReversed); void attemptTrimming(bool doTrimming, char endTrimQV); void attemptTrimming5End(uint32 *errorPos, uint32 endWindow, uint32 endAllowed); void attemptTrimming3End(uint32 *errorPos, uint32 endWindow, uint32 endAllowed); uint32 getClrBgn(void) { return(seqMap[clrBgn]); }; uint32 getClrEnd(void) { return(seqMap[clrEnd]); }; uint32 getSeqLen(void) { return(seqMap[seqLen]); }; void dump(char *label); // Public for the writer. gkFragment fr; mertrimGlobalData *g; mertrimThreadData *t; AS_IID readIID; uint32 seqLen; uint32 allocLen; char *readName; char *origSeq; char *origQlt; char *corrSeq; char *corrQlt; char *verifyName; char *verifySeq; char *verifyErr; uint32 *seqMap; uint32 nMersExpected; uint32 nMersTested; uint32 nMersFound; uint32 nMersCorrect; merStream *rMS; // kmers in the read, for searching against genomic kmers uint32 clrBgn; uint32 clrEnd; bool garbageInInput; bool gapInConfirmedKmers; bool hasNoConfirmedKmers; bool imperfectKmerCoverage; bool containsAdapter; // Read hits adapter kmers uint32 containsAdapterCount; // Number of uncorrected adapter kmers hit uint32 containsAdapterFixed; // Number of corrected adapter kmers hit uint32 containsAdapterBgn; // Location of adapter uint32 containsAdapterEnd; // Location of adapter bool suspectedChimer; uint32 suspectedChimerBgn; uint32 suspectedChimerEnd; uint32 *disconnect; // per base - a hole before this base uint32 *coverage; // per base - mer coverage uint32 *adapter; // per base - mer coverage in adapter kmers uint32 *corrected; // per base - type of correction here existDB *eDB; uint32 nHole; // Number of spaces (between bases) with no mer coverage uint32 nCorr; // Number of bases corrected uint32 nFail; // Number of bases uncorrected because no answer found uint32 nConf; // Number of bases uncorrected because multiple answers found char merstring[256]; logMsg log; }; // Scan the sequence, counting the number of kmers verified. If we find all of them, we're done. // uint32 mertrimComputation::evaluate(void) { if (VERBOSE > 1) log.add("\nPROCESS read %d name %s\n", readIID, readName); if (garbageInInput == true) return(ALLCRAP); if (rMS == NULL) rMS = new merStream(t->kb, new seqStream(corrSeq, seqLen), false, true); rMS->rewind(); nMersExpected = clrEnd - clrBgn - g->merSize + 1; nMersTested = 0; nMersFound = 0; nMersCorrect = 0; while ((rMS->nextMer()) && (rMS->thePositionInSequence() + g->merSize - 1 < clrEnd)) { if (rMS->thePositionInSequence() < clrBgn) // Mer before the clear range begins. continue; if (clrEnd <= rMS->thePositionInSequence() + g->merSize - 1) // Mer after the clear range ends continue; nMersTested++; //log.add("pos %d count %d\n", // rMS->thePositionInSequence() + g->merSize - 1, // eDB->count(rMS->theCMer())); if (eDB->count(rMS->theCMer()) >= g->minCorrect) // We don't need to correct this kmer. nMersCorrect++; if (eDB->count(rMS->theCMer()) >= g->minVerified) // We trust this mer. nMersFound++; } if (VERBOSE > 0) log.add("INITIAL read %u %s len %u has %u mers, %u correct and %u trusted.\n", readIID, readName, seqLen, nMersTested, nMersCorrect, nMersFound); if (nMersCorrect == nMersExpected) // All mers correct, read is 100% verified! return(ALLGOOD); if (nMersFound == 0) { // No trusted kMers found, read is 100% garbage (or 100% unique). hasNoConfirmedKmers = true; return(ALLCRAP); } // Attempt correction. return(ATTEMPTCORRECTION); } void mertrimComputation::reverse(void) { uint32 c = 0; uint32 *s = NULL; uint32 *S = NULL; reverseComplement(corrSeq, corrQlt, seqLen); uint32 cb = seqLen - clrBgn; uint32 ce = seqLen - clrEnd; clrBgn = ce; clrEnd = cb; delete rMS; rMS = new merStream(t->kb, new seqStream(corrSeq, seqLen), false, true); if (corrected) { s = corrected; S = corrected + seqLen - 1; while (s < S) { if (*s == 'X') *s = 0; if (*S == 'X') *S = 0; c = *s; *s++ = *S; *S-- = c; } } if (adapter) { s = adapter; S = adapter + seqLen - 1; while (s < S) { c = *s; *s++ = *S; *S-- = c; } } s = seqMap; S = seqMap + seqLen - 1; while (s < S) { c = *s; *s++ = *S; *S-- = c; } } void mertrimComputation::analyze(void) { if (rMS == NULL) return; rMS->rewind(); if (coverage == NULL) coverage = new uint32 [allocLen]; if (disconnect == NULL) disconnect = new uint32 [allocLen]; memset(coverage, 0, sizeof(uint32) * (allocLen)); memset(disconnect, 0, sizeof(uint32) * (allocLen)); while (rMS->nextMer()) { uint32 posBgn = rMS->thePositionInSequence(); uint32 posEnd = rMS->thePositionInSequence() + g->merSize; assert(posEnd <= seqLen); if (eDB->count(rMS->theCMer()) < g->minVerified) // This mer is too weak for us. SKip it. continue; // If we aren't the first mer, then there should be coverage for our first base. If not, // we have found a correctable error, an uncorrectable error, or a chimeric read. if ((posBgn > 0) && (coverage[posBgn-1] > 0) && (coverage[posBgn] == 0)) disconnect[posBgn-1] = disconnect[posBgn] = 'D'; // Add coverage for the good mer. for (uint32 add=posBgn; addrewind(); if (VERBOSE > 1) dump("ANALYZE"); } bool mertrimComputation::correctMismatch(uint32 pos, uint32 mNum, uint32 mExtra, bool isReversed) { uint32 nA = (corrSeq[pos] != 'A') ? testBaseChange(pos, 'A') : 0; uint32 nC = (corrSeq[pos] != 'C') ? testBaseChange(pos, 'C') : 0; uint32 nG = (corrSeq[pos] != 'G') ? testBaseChange(pos, 'G') : 0; uint32 nT = (corrSeq[pos] != 'T') ? testBaseChange(pos, 'T') : 0; uint32 rB = 0; // Base to change to uint32 rV = 0; // Count of that kmer evidence uint32 nR = 0; if (VERBOSE > 2) { if (nA > mNum + mExtra) log.add("testA at %d -- %d req=%d\n", pos, nA, mNum + mExtra); if (nC > mNum + mExtra) log.add("testC at %d -- %d req=%d\n", pos, nC, mNum + mExtra); if (nG > mNum + mExtra) log.add("testG at %d -- %d req=%d\n", pos, nG, mNum + mExtra); if (nT > mNum + mExtra) log.add("testT at %d -- %d req=%d\n", pos, nT, mNum + mExtra); } // VERBOSE // If we found a single perfectly correct choice, ignore all the other solutions. if (nA == g->merSize) nR++; if (nC == g->merSize) nR++; if (nG == g->merSize) nR++; if (nT == g->merSize) nR++; if (nR == 1) { if (nA != g->merSize) nA = 0; if (nC != g->merSize) nC = 0; if (nG != g->merSize) nG = 0; if (nT != g->merSize) nT = 0; } // Count the number of viable solutions. nR = 0; if (nA > mNum + mExtra) { nR++; rB = 'A'; rV = nA; } if (nC > mNum + mExtra) { nR++; rB = 'C'; rV = nC; } if (nG > mNum + mExtra) { nR++; rB = 'G'; rV = nG; } if (nT > mNum + mExtra) { nR++; rB = 'T'; rV = nT; } if (nR == 0) // Nothing viable, keep the base as is. return(false); // Something to change to. By definition, this is a stronger mer than the original. if (nR > 1) { // Multiple solutions. Pick the most common. If we don't do this, the confirmed kmer // coverage drops at this location, and we trim the read. This would result in // zero coverage at every variation. uint32 mm = MAX(MAX(nA, nC), MAX(nG, nT)); if (nA == mm) { rB = 'A'; rV = nA; } if (nC == mm) { rB = 'C'; rV = nC; } if (nG == mm) { rB = 'G'; rV = nG; } if (nT == mm) { rB = 'T'; rV = nT; } //corrected[pos] = 'X'; //return(false); } // One solution! Correct it. if (VERBOSE > 0) { if (nR > 1) log.add("Correct read %d at position %d from %c (%u) to %c (%u) (QV %d) (%s) (multiple choices nA=%d nC=%d nG=%d nT=%d)\n", readIID, (isReversed == false) ? pos : seqLen - pos, corrSeq[pos], mNum, rB, rV, corrQlt[pos], (isReversed == false) ? "fwd" : "rev", nA, nC, nG, nT); else log.add("Correct read %d at position %d from %c (%u) to %c (%u) (QV %d) (%s)\n", readIID, (isReversed == false) ? pos : seqLen - pos, corrSeq[pos], mNum, rB, rV, corrQlt[pos], (isReversed == false) ? "fwd" : "rev"); } // VERBOSE corrSeq[pos] = rB; corrected[pos] = 'C'; // Rebuild the merStream to use the corrected sequence, then move to the same position in // the sequence. This is done because we cannot simply change the string -- we need to // change the state of the kMerBuilder associated with the merStream, and we can't do that. #ifdef USE_MERSTREAM_REBUILD rMS->rebuild(); #else pos = rMS->thePositionInSequence(); delete rMS; rMS = new merStream(t->kb, new seqStream(corrSeq, seqLen), false, true); rMS->nextMer(); while (pos != rMS->thePositionInSequence()) rMS->nextMer(); #endif return(true); } bool mertrimComputation::correctIndel(uint32 pos, uint32 mNum, uint32 mExtra, bool isReversed) { uint32 nD = testBaseIndel(pos, '-'); uint32 nA = testBaseIndel(pos, 'A'); uint32 nC = testBaseIndel(pos, 'C'); uint32 nG = testBaseIndel(pos, 'G'); uint32 nT = testBaseIndel(pos, 'T'); char rB = 0; uint32 rV = 0; uint32 nR = 0; if (nD > mNum + mExtra) { nR++; rB = '-'; rV = nD; } if (nA > mNum + mExtra) { nR++; rB = 'A'; rV = nA; } if (nC > mNum + mExtra) { nR++; rB = 'C'; rV = nC; } if (nG > mNum + mExtra) { nR++; rB = 'G'; rV = nG; } if (nT > mNum + mExtra) { nR++; rB = 'T'; rV = nT; } if (VERBOSE > 2) { if (nD > mNum + mExtra) log.add("test-- %d -- %d req=%d\n", pos, nD, mNum + mExtra); if (nA > mNum + mExtra) log.add("test+A %d -- %d req=%d\n", pos, nA, mNum + mExtra); if (nC > mNum + mExtra) log.add("test+C %d -- %d req=%d\n", pos, nC, mNum + mExtra); if (nG > mNum + mExtra) log.add("test+G %d -- %d req=%d\n", pos, nG, mNum + mExtra); if (nT > mNum + mExtra) log.add("test+T %d -- %d req=%d\n", pos, nT, mNum + mExtra); } // VERBOSE if (nR == 0) // No solutions. return(false); if (nR > 1) { // Multiple solutions. corrected[pos] = 'X'; return(false); } // One solution. Make a correction. Either a deletion or an insert. if (nD > mNum + mExtra) { if (VERBOSE > 0) { log.add("Correct read %d at position %d from %c (%u) to DELETE (%u) (QV %d) (%s)\n", readIID, (isReversed == false) ? pos : seqLen - pos, corrSeq[pos], mNum, rV, corrQlt[pos], (isReversed == false) ? "fwd" : "rev"); } // VERBOSE for (uint32 i=pos; i 0) { log.add("Correct read %d at position %d from . (%u) to INSERT %c (%u) (%s)\n", readIID, (isReversed == false) ? pos : seqLen - pos, mNum, rB, rV, (isReversed == false) ? "fwd" : "rev"); } // VERBOSE for (uint32 i=seqLen+1; i>pos; i--) { corrSeq[i] = corrSeq[i-1]; corrQlt[i] = corrQlt[i-1]; if (adapter) adapter[i] = adapter[i-1]; seqMap[i] = seqMap[i-1]; } corrSeq[pos] = rB; corrQlt[pos] = '5'; if (adapter) adapter[pos] = 0; seqMap[pos] = seqMap[pos-1]; seqLen++; clrEnd++; corrected[pos] = 'I'; } // Rebuild the merstream. When we call analyze() the stream gets rewound. If we do not // restore our position, it is possible to get stuck in an infinite loop, inserting and // deleting the same base over and over. This happens because we don't explicitly require // that the mer we are at be found, just that we find enough mers to make the change. // // So, on the next pass through, we'd encounter the same mer we didn't find before, attempt // to change it again, and possibly delete the base we inserted. // // test+A 57 -- 6 req=2 // Correct read 6 at position 57 INSERT A // // test-- 58 -- 3 req=2 // Correct read 6 at position 58 from A to DELETE (QV 5) // // The first time through, we insert an A (with 6 mers agreeing). The second time through, // since we didn't fix the mer we were at, our choice is to delete the base we inserted (3 // mers tell us to do so). #ifdef USE_MERSTREAM_REBUILD rMS->rebuild(); #else pos = rMS->thePositionInSequence(); delete rMS; rMS = new merStream(t->kb, new seqStream(corrSeq, seqLen), false, true); analyze(); rMS->nextMer(); while (pos != rMS->thePositionInSequence()) rMS->nextMer(); #endif return(true); } void mertrimComputation::searchAdapter(bool isReversed) { if (corrected == NULL) { corrected = new uint32 [allocLen]; memset(corrected, 0, sizeof(uint32) * (allocLen)); } if (rMS == NULL) rMS = new merStream(t->kb, new seqStream(corrSeq, seqLen), false, true); rMS->rewind(); // A combination of evaluate() and attemptCorrection(). Find any adapter kmers in the // read, do any corrections, then mark (in array 'adapter' the location of those adapter // bases while (rMS->nextMer()) { uint32 pos = rMS->thePositionInSequence() + g->merSize - 1; uint32 count = eDB->count(rMS->theCMer()); if (count >= 1) { // Mer exists, no need to correct. containsAdapter = true; continue; } uint32 mNum = testBaseChange(pos, corrSeq[pos]); // Test if we can repair the sequence with a single base change. if (g->correctMismatch) if (correctMismatch(pos, mNum, 1, isReversed)) { containsAdapter = true; containsAdapterFixed++; } } } void mertrimComputation::scoreAdapter(void) { if (containsAdapter == false) return; assert(adapter == NULL); adapter = new uint32 [allocLen]; memset(adapter, 0, sizeof(uint32) * (allocLen)); assert(clrBgn == 0); assert(clrEnd == seqLen); containsAdapterBgn = seqLen; containsAdapterEnd = 0; rMS->rewind(); while (rMS->nextMer()) { uint32 bgn = rMS->thePositionInSequence(); uint32 end = bgn + g->merSize - 1; uint32 count = eDB->count(rMS->theCMer()); if (count == 0) continue; containsAdapterCount++; containsAdapterBgn = MIN(containsAdapterBgn, bgn); containsAdapterEnd = MAX(containsAdapterEnd, end + 1); if (VERBOSE > 1) log.add("ADAPTER at "F_U32","F_U32" ["F_U32","F_U32"]\n", bgn, end, containsAdapterBgn, containsAdapterEnd); for (uint32 a=bgn; a<=end; a++) adapter[a]++; } if (VERBOSE) dump("ADAPTERSEARCH"); } void mertrimComputation::attemptCorrection(bool isReversed) { if (corrected == NULL) { corrected = new uint32 [allocLen]; memset(corrected, 0, sizeof(uint32) * (allocLen)); } assert(coverage); assert(disconnect); assert(corrected); assert(rMS != NULL); rMS->rewind(); while (rMS->nextMer()) { uint32 pos = rMS->thePositionInSequence() + g->merSize - 1; uint32 count = eDB->count(rMS->theCMer()); //log.add("MER at %d is %s has count %d %s\n", // pos, // rMS->theFMer().merToString(merstring), // (count >= g->minCorrect) ? "CORRECT" : "ERROR", // count); if (count >= g->minCorrect) // Mer is correct, no need to correct it! continue; // State the minimum number of mers we'd accept as evidence any change we make is correct. The // penalty for OVER correcting (too low a threshold) is potentially severe -- we could // insert/delete over and over and over eventually blowing up. The penalty for UNDER // correcting, however, is that we trim a read too aggressively. // // Being strictly greater than before works for mismatches and deletions. // // For insertions, especially insertions in single nucleotide runs, this doesn't work so well. // There are two cases, an insertion before a run, and an insertion after a run. Below, X // represents ACGT, and the A's are the run. // XXXXXAAAAA: inserting an A after the X's, returns the same sequence, and the count of // good mers doesn't change. // AAAAAXXXXX: inserting an A before the X's changes the sequence, returning AAAAAAXXXX, // and it is possible (likely) that this new sequence will have a higher count. // // We therefore require that we find at least TWO more good mers before accepting a change. // // The drawback of this is that we cannot correct two adjacent errors. The first error // (the one we're currently working on) is corrected and adds one to the coverage count, // but then we hit that second error and do not find any more mers. // // A solution would be to retry any base we cannot correct and allow a positive change of // one mer to accept the change. (in other words, change +1 below to +0). uint32 mNum = testBaseChange(pos, corrSeq[pos]); // Test if we can repair the sequence with a single base change. if (g->correctMismatch) if (correctMismatch(pos, mNum, 1, isReversed)) continue; if ((g->correctIndel) && (g->merSize < pos) && (pos < seqLen - g->merSize)) if (correctIndel(pos, mNum, 3, isReversed)) continue; } if (VERBOSE > 1) { dump("POSTCORRECT"); } // VERBOSE } uint32 mertrimComputation::testBases(char *bases, uint32 basesLen) { uint32 offset = 0; uint32 numConfirmed = 0; // // UNTESTED with KMER_WORDS != 1 // kMer F(g->merSize); kMer R(g->merSize); for (uint32 i=1; imerSize && offsetmerSize && offsetcount(F) >= g->minVerified) numConfirmed++; } else { if (eDB->count(R) >= g->minVerified) numConfirmed++; } } return(numConfirmed); } // Attempt to change the base at pos to make the kmers spanning it agree. // Returns the number of kmers validated, and the letter to change to. // uint32 mertrimComputation::testBaseChange(uint32 pos, char replacement) { uint32 numConfirmed = 0; char originalBase = corrSeq[pos]; uint32 offset = pos + 1 - g->merSize; corrSeq[pos] = replacement; numConfirmed = testBases(corrSeq + offset, MIN(seqLen - offset, 2 * g->merSize - 1)); #ifdef TEST_TESTBASE { uint32 oldConfirmed = 0; merStream *localms = new merStream(new kMerBuilder(merSize, compression, 0L), new seqStream(corrSeq + offset, seqLen - offset), true, true); // Test for (uint32 i=0; imerSize && localms->nextMer(); i++) if (eDB->count(localms->theCMer()) >= g->minVerified) oldConfirmed++; delete localms; assert(oldConfirmed == numConfirmed); } #endif corrSeq[pos] = originalBase; //if (numConfirmed > 0) // log.add("testBaseChange() pos=%d replacement=%c confirmed=%d\n", // pos, replacement, numConfirmed); return(numConfirmed); } uint32 mertrimComputation::testBaseIndel(uint32 pos, char replacement) { uint32 numConfirmed = 0; char testStr[128] = {0}; uint32 len = 0; uint32 offset = pos + 1 - g->merSize; uint32 limit = g->merSize * 2 - 1; assert(2 * g->merSize < 120); // Overly pessimistic // Copy the first merSize bases. while (len < g->merSize - 1) testStr[len++] = corrSeq[offset++]; // Copy the second merSize bases, but overwrite the last base in the first copy (if we're testing // a de;etion) or insert a replacement base. if (replacement == '-') { offset++; } else { testStr[len++] = replacement; } // Copy the rest of the bases. while ((len < limit) && (corrSeq[offset])) testStr[len++] = corrSeq[offset++]; numConfirmed = testBases(testStr, len); #ifdef TEST_TESTBASE { uint32 oldConfirmed = 0; merStream *localms = new merStream(new kMerBuilder(merSize, compression, 0L), new seqStream(testStr, len), true, true); // Test for (uint32 i=0; imerSize && localms->nextMer(); i++) if (existDB->count(localms->theCMer()) >= g->minVerified) oldConfirmed++; delete localms; assert(oldConfirmed == numConfirmed); } #endif //if (numConfirmed > 0) // log.add("testBaseIndel() pos=%d replacement=%c confirmed=%d\n", // pos, replacement, numConfirmed); return(numConfirmed); } void mertrimComputation::attemptTrimming5End(uint32 *errorPos, uint32 endWindow, uint32 errAllow) { for (bool doTrim = true; doTrim; ) { uint32 endFound = 0; uint32 endTrimPos = 0; uint32 bgn = clrBgn; uint32 end = clrBgn + endWindow; if (end > clrEnd) end = clrEnd; for (uint32 i=bgn; i 1) log.add("BGNTRIM found=%u pos=%u from %u to %u\n", endFound, endTrimPos, clrBgn, clrBgn + endWindow); if (endFound > errAllow) clrBgn = endTrimPos; else doTrim = false; } } void mertrimComputation::attemptTrimming3End(uint32 *errorPos, uint32 endWindow, uint32 errAllow) { for (bool doTrim = true; doTrim; ) { uint32 endFound = 0; uint32 endTrimPos = clrEnd; uint32 bgn = clrBgn; uint32 end = clrEnd; if (clrBgn + endWindow <= clrEnd) bgn = clrEnd - endWindow; assert(bgn >= clrBgn); assert(bgn <= clrEnd); for (int32 i=bgn; i 1) log.add("ENDTRIM found=%u pos=%u from %u to %u\n", endFound, endTrimPos, clrEnd - endWindow, clrEnd); if (endFound > errAllow) clrEnd = endTrimPos; else doTrim = false; } } void mertrimComputation::attemptTrimming(bool doTrimming, char endTrimQV) { // Just bail if the read is all junk. Nothing to do here. // if ((garbageInInput) || (hasNoConfirmedKmers)) { clrBgn = 0; clrEnd = 0; return; } if ((clrBgn == 0) && (clrEnd == 0)) return; assert(coverage != NULL); // Lop off the ends with no confirmed kmers or a QV less than 3. The Illumina '2' qv ('B' in // Illumina 1.5+ encodings) is defined as "rest of read is < QV 15; do not use". // if (doTrimming) { while ((clrBgn < clrEnd) && ((coverage[clrBgn] == 0) || (corrQlt[clrBgn] <= endTrimQV))) clrBgn++; while ((clrEnd > clrBgn) && ((coverage[clrEnd-1] == 0) || (corrQlt[clrEnd-1] <= endTrimQV))) clrEnd--; } //log.add("TRIM: %d,%d (lop-off-ends)\n", clrBgn, clrEnd); //dump("TRIM"); // Deal with adapter. We'll pick the biggest end that is adapter free for our sequence. // If both ends have adapter, so be it; the read gets trashed. if (containsAdapter) { containsAdapterBgn = seqLen; containsAdapterEnd = 0; for (uint32 i=0; i 0) && (i < containsAdapterBgn)) containsAdapterBgn = i; if ((adapter[i] > 0) && (containsAdapterEnd < i)) containsAdapterEnd = i + 1; } if (containsAdapterBgn < clrBgn) containsAdapterBgn = clrBgn; if (clrEnd < containsAdapterEnd) containsAdapterEnd = clrEnd; uint32 bgnClear = containsAdapterBgn - clrBgn; uint32 endClear = clrEnd - containsAdapterEnd; if (bgnClear > endClear) // Start is bigger clrEnd = containsAdapterBgn; else // End is bigger clrBgn = containsAdapterEnd; if (clrBgn >= clrEnd) { clrBgn = 0; clrEnd = 0; } //log.add("TRIM: %d,%d (adapter)\n", clrBgn, clrEnd); //dump("TRIM"); } if (doTrimming == false) return; // Lop off the ends with no confirmed kmers (again) // while ((clrBgn < clrEnd) && (coverage[clrBgn] == 0)) clrBgn++; while ((clrEnd > clrBgn) && (coverage[clrEnd-1] == 0)) clrEnd--; //log.add("TRIM: %d,%d (lop-off-ends-2)\n", clrBgn, clrEnd); //dump("TRIM"); // True if there is an error at position i. // uint32 *errorPos = new uint32 [seqLen]; for (uint32 i=0; iendTrimNum; i++) { attemptTrimming5End(errorPos, g->merSize * g->endTrimWinScale[i], g->endTrimErrAllow[i]); attemptTrimming3End(errorPos, g->merSize * g->endTrimWinScale[i], g->endTrimErrAllow[i]); } delete [] errorPos; // If there are zero coverage areas in the interior, trash the whole read. if (g->discardZeroCoverage == true) { for (uint32 i=clrBgn; idiscardImperfectCoverage == true) && (clrBgn < clrEnd) && (clrEnd > 0)) { uint32 bgn = clrBgn; uint32 end = clrEnd - 1; while ((bgn + 1 < end) && (coverage[bgn] < coverage[bgn+1])) bgn++; while ((bgn + 1 < end) && (coverage[end-1] > coverage[end])) end--; uint32 bgnc = coverage[bgn]; uint32 endc = coverage[end]; if (VERBOSE) log.add("IMPERFECT: bgn=%u %u end=%u %u\n", bgn, bgnc, end, endc); if (bgnc != endc) imperfectKmerCoverage = true; else for (uint32 i=bgn; i<=end; i++) if (coverage[i] != bgnc) imperfectKmerCoverage = true; } // If the coverage isn't perfect (see above), trim out the ends that make it imperfect. // This leaves interior imperfect regions, but there should be enough to get // an overlap on either end. if ((g->trimImperfectCoverage == true) && (clrBgn < clrEnd) && (clrEnd > 0)) { uint32 bgn = clrBgn; uint32 end = clrEnd - 1; // Search from the ends in until we find the highest coverage. while ((bgn <= end) && (coverage[bgn] < g->merSize - 4)) bgn++; while ((bgn <= end) && (coverage[end] < g->merSize - 4)) end--; end++; // Check that everything between is nice and high. Why the -2? This lets // us tolerate one or two 'missing' kmers. uint32 mbgn = bgn; // Maximal bgn/end region uint32 mend = bgn; uint32 tbgn = bgn; // Currently testing region uint32 tend = bgn; while (tend < end) { if (coverage[tend] < g->merSize - 4) { if ((mend - mbgn) < (tend - tbgn)) { mbgn = tbgn; mend = tend; } tbgn = tend + 1; // Next region starts on the next position. } tend++; } // If we found a sub region, reset to it (after remembering to check the last sub region). if (mbgn < mend) { if ((mend - mbgn) < (tend - tbgn)) { mbgn = tbgn; mend = tend; } if (VERBOSE) log.add("Reset clr from %d,%d to %d,%d\n", bgn, end, mbgn, mend); bgn = mbgn; end = mend; } // Then search towards the ends, as long as the coverage is strictly decreasing. if (bgn < end) { while ((bgn > clrBgn) && (coverage[bgn-1] < coverage[bgn]) && (coverage[bgn-1] > 0)) bgn--; while ((end < clrEnd) && (coverage[end] < coverage[end-1]) && (coverage[end] > 0)) end++; } else { imperfectKmerCoverage = true; } clrBgn = bgn; clrEnd = end; } // Make sense of the clear ranges. If they're invalid, reset to 0,0. if (clrBgn >= clrEnd) { clrBgn = 0; clrEnd = 0; } if (VERBOSE > 1) { log.add("TRIM: %d,%d (post)\n", clrBgn, clrEnd); dump("TRIM"); } // VERBOSE } void mertrimComputation::dump(char *label) { char *logLine = new char [4 * seqLen]; uint32 logPos = 0; uint32 bogus = (clrEnd == 0) ? 0 : UINT32_MAX; log.add("%s read %d %s len %d (trim %d-%d)\n", label, readIID, readName, seqLen, clrBgn, clrEnd); logPos = 0; for (uint32 i=0; origSeq[i]; i++) { if (i == clrBgn) { logLine[logPos++] = '-'; logLine[logPos++] = '['; } if (i == bogus) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; } // read deleted logLine[logPos++] = origSeq[i]; if (i+1 == clrEnd) { logLine[logPos++] = ']'; logLine[logPos++] = '-'; } } strcpy(logLine + logPos, " (ORI)\n"); log.add(logLine); logPos = 0; for (uint32 i=0; it = t; s->eDB = g->genomicDB; uint32 eval = s->evaluate(); // Attempt correction if there are kmers to correct from. if (eval == ATTEMPTCORRECTION) { s->analyze(); s->attemptCorrection(false); s->reverse(); s->analyze(); s->attemptCorrection(true); s->reverse(); } // Search for linker/adapter. This needs to be after correction, since indel screws // up the clear ranges we set in scoreAdapter(). if ((eval != ALLCRAP) && (g->adapterDB != NULL)) { s->eDB = g->adapterDB; s->searchAdapter(false); s->reverse(); s->searchAdapter(true); s->reverse(); s->scoreAdapter(); s->eDB = g->genomicDB; } // Attempt trimming if the read wasn't perfect if (eval != ALLGOOD) { s->analyze(); s->attemptTrimming(g->doTrimming, g->endTrimQV); } if (VERBOSE) s->dump("FINAL"); } mertrimComputation * mertrimReaderGatekeeper(mertrimGlobalData *g) { mertrimComputation *s = NULL; while ((g->gktCur <= g->gktEnd) && (s == NULL)) { s = new mertrimComputation(); g->gkRead->gkStore_getFragment(g->gktCur, &s->fr, GKFRAGMENT_QLT); g->gktCur++; if ((g->forceCorrection) || (g->gkRead->gkStore_getLibrary(s->fr.gkFragment_getLibraryIID())->doTrim_initialMerBased)) { s->initializeGatekeeper(g); } else { delete s; s = NULL; } } return(s); } mertrimComputation * mertrimReaderFASTQ(mertrimGlobalData *g) { mertrimComputation *s = new mertrimComputation(); if (s->initializeFASTQ(g) == false) { delete s; s = NULL; } return(s); } void * mertrimReader(void *G) { mertrimGlobalData *g = (mertrimGlobalData *)G; mertrimComputation *s = NULL; if (g->gkRead) s = mertrimReaderGatekeeper(g); if (g->fqInput) s = mertrimReaderFASTQ(g); return(s); } void mertrimWriterGatekeeper(mertrimGlobalData *g, mertrimComputation *s) { mertrimResult res; res.readIID = s->fr.gkFragment_getReadIID(); if ((s->getClrEnd() <= s->getClrBgn()) || (s->getClrEnd() - s->getClrBgn() < AS_READ_MIN_LEN)) res.deleted = true; else res.deleted = false; res.clrBgn = s->getClrBgn(); res.clrEnd = s->getClrEnd(); res.chimer = s->suspectedChimer; res.chmBgn = s->suspectedChimerBgn; res.chmEnd = s->suspectedChimerEnd; res.writeResult(g->resFile); } void mertrimWriterFASTQ(mertrimGlobalData *g, mertrimComputation *s) { // Note that getClrBgn/getClrEnd return positions in the ORIGINAL read, not the correct read. // DO NOT USE HERE! uint32 seqOffset = 0; char label[256]; label[0] = 0; if (s->garbageInInput == true) { strcat(label, "DEL-GARBAGE"); seqOffset = 0; s->corrSeq[0] = 0; s->corrQlt[0] = 0; s->clrBgn = 0; s->clrEnd = 0; goto outputFastq; } if (s->hasNoConfirmedKmers == true) { strcat(label, "DEL-NO-KMER"); seqOffset = 0; s->corrSeq[0] = 0; s->corrQlt[0] = 0; s->clrBgn = 0; s->clrEnd = 0; goto outputFastq; } if ((s->containsAdapter == true) && (s->suspectedChimer == false)) { strcat(label, "ADAPTERTRIM"); seqOffset = s->clrBgn; s->corrSeq[s->clrEnd] = 0; s->corrQlt[s->clrEnd] = 0; goto outputFastq; } if ((s->containsAdapter == false) && (s->suspectedChimer == true)) { if (s->suspectedChimerBgn - s->clrBgn >= AS_READ_MIN_LEN) { strcpy(label, "MP"); // Junction read, longest portion would make an MP pair seqOffset = s->clrBgn; s->corrSeq[s->suspectedChimerBgn] = 0; s->corrQlt[s->suspectedChimerBgn] = 0; } else if (s->clrEnd - s->suspectedChimerEnd >= AS_READ_MIN_LEN) { strcpy(label, "PE"); // Junction read, longest portion would make a PE pair seqOffset = s->suspectedChimerEnd; s->corrSeq[s->clrEnd] = 0; s->corrQlt[s->clrEnd] = 0; } else { strcpy(label, "MP-PE-SHORT");// Junction read, neither side is long enough to save, so save nothing. seqOffset = 0; s->corrSeq[0] = 0; s->corrQlt[0] = 0; s->clrBgn = 0; s->clrEnd = 0; } goto outputFastq; } if (s->gapInConfirmedKmers == true) { strcat(label, "DEL-ZERO-COV"); seqOffset = 0; s->corrSeq[0] = 0; s->corrQlt[0] = 0; s->clrBgn = 0; s->clrEnd = 0; goto outputFastq; } if (s->imperfectKmerCoverage == true) { strcat(label, "DEL-INPERFECT-COV"); seqOffset = 0; s->corrSeq[0] = 0; s->corrQlt[0] = 0; s->clrBgn = 0; s->clrEnd = 0; goto outputFastq; } if ((s->suspectedChimer == false) && (s->containsAdapter == false)) { strcat(label, "CLEAN"); seqOffset = s->clrBgn; s->corrSeq[s->clrEnd] = 0; s->corrQlt[s->clrEnd] = 0; goto outputFastq; } assert(s->suspectedChimer); assert(s->containsAdapter); strcpy(label, "JUNCTION+ADAPTER"); seqOffset = 0; s->corrSeq[0] = 0; s->corrQlt[0] = 0; outputFastq: // If there is a verify sequence, verify. uint32 nVerifyErrors = 0; #warning unfinished verify if (s->verifySeq) { } fprintf(g->fqLog->file(), F_U32"\t"F_U32"\tchimer\t%c\t"F_U32"\t"F_U32"\tadapter\t%c\t"F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\t%s\t%s\n", s->clrBgn, s->clrEnd, s->suspectedChimer ? 't' : 'f', s->suspectedChimerBgn, s->suspectedChimerEnd, s->containsAdapter ? 't' : 'f', s->containsAdapterCount, s->containsAdapterFixed, s->containsAdapterBgn, s->containsAdapterEnd, label, s->readName); // Convert from CA QV to Sanger QV for (uint32 i=0; s->corrQlt[i]; i++) s->corrQlt[i] += '!'; // If no sequence, write a single N, with low QV (! == lowest). if ((s->corrSeq[seqOffset] == 0) || (s->corrQlt[seqOffset] == 0)) fprintf(g->fqOutput->file(), "%s type=%s\nN\n+\n!\n", s->readName, label); else fprintf(g->fqOutput->file(), "%s type=%s\n%s\n+\n%s\n", s->readName, label, s->corrSeq + seqOffset, s->corrQlt + seqOffset); if (VERBOSE) s->log.add("RESULT read %d len %d (trim %d-%d) %s\n", s->readIID, s->seqLen, s->clrBgn, s->clrEnd, label); s->log.fwrite(stdout); } void mertrimWriter(void *G, void *S) { mertrimGlobalData *g = (mertrimGlobalData *)G; mertrimComputation *s = (mertrimComputation *)S; // The get*() functions return positions in the original uncorrected sequence. They // map from positions in the corrected sequence (which has inserts and deletes) back // to the original sequence. // assert(s->getClrBgn() <= s->getClrEnd()); assert(s->getClrEnd() <= s->getSeqLen()); if (g->resFile) mertrimWriterGatekeeper(g, s); if (g->fqOutput) mertrimWriterFASTQ(g, s); delete s; } int main(int argc, char **argv) { mertrimGlobalData *g = new mertrimGlobalData; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-g") == 0) { g->gkpPath = argv[++arg]; } else if (strcmp(argv[arg], "-F") == 0) { g->fqInputPath = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { g->fqVerifyPath = argv[++arg]; } else if (strcmp(argv[arg], "-m") == 0) { g->merSize = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-mc") == 0) { g->merCountsFile = argv[++arg]; } else if (strcmp(argv[arg], "-enablecache") == 0) { g->merCountsCache = true; } else if (strcmp(argv[arg], "-mC") == 0) { g->adapCountsFile = argv[++arg]; } else if (strcmp(argv[arg], "-mCillumina") == 0) { g->adapIllumina = 1; } else if (strcmp(argv[arg], "-mC454") == 0) { g->adap454 = 2; } else if (strcmp(argv[arg], "-t") == 0) { g->numThreads = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-b") == 0) { g->gktBgn = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-e") == 0) { g->gktEnd = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-v") == 0) { g->beVerbose = true; } else if (strcmp(argv[arg], "-V") == 0) { VERBOSE++; } else if (strcmp(argv[arg], "-coverage") == 0) { g->actualCoverage = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-correct") == 0) { g->minCorrectFraction = atof(argv[++arg]); if (g->minCorrectFraction >= 1) { g->minCorrect = (uint32)g->minCorrectFraction; g->minCorrectFraction = 0; } } else if (strcmp(argv[arg], "-evidence") == 0) { g->minVerifiedFraction = atof(argv[++arg]); if (g->minVerifiedFraction >= 1) { g->minVerified = (uint32)g->minVerifiedFraction; g->minVerifiedFraction = 0; } } else if (strcmp(argv[arg], "-endtrim") == 0) { if (g->endTrimDefault == true) g->endTrimNum = 0; g->endTrimWinScale[g->endTrimNum] = atof(argv[++arg]); g->endTrimErrAllow[g->endTrimNum] = atoi(argv[++arg]); g->endTrimNum++; } else if (strcmp(argv[arg], "-notrimming") == 0) { g->doTrimming = false; } else if (strcmp(argv[arg], "-discardzero") == 0) { g->discardZeroCoverage = true; } else if (strcmp(argv[arg], "-discardimperfect") == 0) { g->discardImperfectCoverage = true; } else if (strcmp(argv[arg], "-notrimimperfect") == 0) { g->trimImperfectCoverage = false; } else if (strcmp(argv[arg], "-endtrimqv") == 0) { g->endTrimQV = argv[++arg][0]; } else if (strcmp(argv[arg], "-f") == 0) { g->forceCorrection = true; } else if (strcmp(argv[arg], "-NM") == 0) { g->correctMismatch = false; } else if (strcmp(argv[arg], "-NI") == 0) { g->correctIndel = false; } else if (strcmp(argv[arg], "-o") == 0) { g->fqOutputPath = argv[++arg]; g->resPath = argv[arg]; } else { fprintf(stderr, "unknown option '%s'\n", argv[arg]); err++; } arg++; } if ((g->gkpPath == 0L) && (g->fqInputPath == 0L)) err++; if (err) { fprintf(stderr, "usage: %s ...\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " -F reads.fastq input reads\n"); fprintf(stderr, " -o reads.fastq output reads\n"); fprintf(stderr, "\n"); fprintf(stderr, " -T reads.fasta truth reads for validation\n"); fprintf(stderr, "\n"); fprintf(stderr, " -m ms mer size\n"); fprintf(stderr, " -mc counts kmer database (in 'counts.mcdat' and 'counts.mcidx')\n"); fprintf(stderr, " -enablecache dump the final kmer data to 'counts.merTrimDB'\n"); fprintf(stderr, "\n"); fprintf(stderr, " -coverage C\n"); fprintf(stderr, " -correct n mers with count below n can be changed\n"); fprintf(stderr, " (that is, count >= n are correct mers)\n"); fprintf(stderr, " -evidence n mers with count at least n will be used for changes\n"); fprintf(stderr, "\n"); fprintf(stderr, " -mC adapter.fasta screen for these adapter sequences\n"); fprintf(stderr, " -mCillumina screen for common Illumina adapter sequences\n"); fprintf(stderr, " -mC454 screen for common 454 adapter and linker sequences\n"); fprintf(stderr, "\n"); fprintf(stderr, " -endtrim (undocumented)\n"); fprintf(stderr, " -notrimming do only correction, no trimming\n"); fprintf(stderr, " -discardzero trash the whole read if coverage drops to zero in the middle\n"); fprintf(stderr, " -discardimperfect trash the whole read if coverage isn't perfect\n"); fprintf(stderr, " -notrimimperfect do NOT trim off ends that make the coverage imperfect\n"); fprintf(stderr, " -endtrimqv Q trim ends of reads if they are below qv Q (Sanger encoded; default '2')\n"); fprintf(stderr, "\n"); fprintf(stderr, " -NM do NOT correct mismatch errors\n"); fprintf(stderr, " -NI do NOT correct indel errors\n"); fprintf(stderr, "\n"); fprintf(stderr, " -t T use T CPU cores\n"); fprintf(stderr, "\n"); fprintf(stderr, " -v report progress to stderr\n"); fprintf(stderr, " -V report trimming evidence to stdout (more -V -> more reports)\n"); fprintf(stderr, "\n"); exit(1); } gkpStoreFile::registerFile(); g->initialize(); gkFragment fr; #if 0 // DEBUG, non-threaded version. speedCounter SC(" Trimming: %11.0f reads -- %7.5f reads/second\r", 1.0, 0x1fff, true); mertrimThreadData *t = new mertrimThreadData(g); g->tBgn = 140222; g->tCur = 140222; g->tEnd = 140222; mertrimComputation *s = (mertrimComputation *)mertrimReader(g); while (s) { mertrimWorker(g, t, s); mertrimWriter(g, s); SC.tick(); s = (mertrimComputation *)mertrimReader(g); } delete t; #else // PRODUCTION, threaded version sweatShop *ss = new sweatShop(mertrimReader, mertrimWorker, mertrimWriter); ss->setLoaderQueueSize(16384); ss->setWriterQueueSize(1024); ss->setNumberOfWorkers(g->numThreads); for (uint32 w=0; wnumThreads; w++) ss->setThreadData(w, new mertrimThreadData(g)); // these leak ss->run(g, g->beVerbose); // true == verbose #endif delete g; fprintf(stderr, "\nSuccess! Bye.\n"); return(0); } canu-1.3/src/merTrim/merTrimAdapter.C000066400000000000000000000573441272335422300175550ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_MER/merTrimAdapter.C * * Modifications by: * * Brian P. Walenz from 2012-MAY-10 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "bio++.H" #include "sweatShop.H" #include "existDB.H" #include "positionDB.H" #include "libmeryl.H" #include "logMsg.H" // Size of the begin/end substring to use when creating end-to-end sequences. This must be larger // than 2*merSize-1, so there is a kmer spanning the junction on both sides. Larger is not a // problem; it slightly slows down the table build. // #define KSIZE 32 char * createAdapterString(bool adapIllumina, bool adap454) { uint32 na = 0; uint32 adapterMax = 256; const char *adapterNam[256]; const char *adapterSeq[256]; if (adapIllumina) { adapterNam[na] = "Illumina Single End Adapter 1"; adapterSeq[na++] = "ACACTCTTTCCCTACACGACGCTGTTCCATCT"; adapterNam[na] = "Illumina Single End Adapter 2"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGCTCTTCCGATCT"; adapterNam[na] = "Illumina Single End PCR Primer 1"; adapterSeq[na++] = "AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTCCGATCT"; adapterNam[na] = "Illumina Single End PCR Primer 2"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGCTCTTCCGATCT"; adapterNam[na] = "Illumina Single End Sequencing Primer"; adapterSeq[na++] = "ACACTCTTTCCCTACACGACGCTCTTCCGATCT"; adapterNam[na] = "Illumina Paired End Adapter 1"; adapterSeq[na++] = "ACACTCTTTCCCTACACGACGCTCTTCCGATCT"; adapterNam[na] = "Illumina Paired End Adapter 2"; adapterSeq[na++] = "CTCGGCATTCCTGCTGAACCGCTCTTCCGATCT"; adapterNam[na] = "Illumina Paried End PCR Primer 1"; adapterSeq[na++] = "AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTCCGATCT"; adapterNam[na] = "Illumina Paired End PCR Primer 2"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCGGTCTCGGCATTCCTGCTGAACCGCTCTTCCGATCT"; adapterNam[na] = "Illumina Paried End Sequencing Primer 1"; adapterSeq[na++] = "ACACTCTTTCCCTACACGACGCTCTTCCGATCT"; adapterNam[na] = "Illumina Paired End Sequencing Primer 2"; adapterSeq[na++] = "CGGTCTCGGCATTCCTACTGAACCGCTCTTCCGATCT"; adapterNam[na] = "Illumina DpnII expression Adapter 1"; adapterSeq[na++] = "ACAGGTTCAGAGTTCTACAGTCCGAC"; adapterNam[na] = "Illumina DpnII expression Adapter 2"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina DpnII expression PCR Primer 1"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina DpnII expression PCR Primer 2"; adapterSeq[na++] = "AATGATACGGCGACCACCGACAGGTTCAGAGTTCTACAGTCCGA"; adapterNam[na] = "Illumina DpnII expression Sequencing Primer"; adapterSeq[na++] = "CGACAGGTTCAGAGTTCTACAGTCCGACGATC"; adapterNam[na] = "Illumina NlaIII expression Adapter 1"; adapterSeq[na++] = "ACAGGTTCAGAGTTCTACAGTCCGACATG"; adapterNam[na] = "Illumina NlaIII expression Adapter 2"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina NlaIII expression PCR Primer 1"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina NlaIII expression PCR Primer 2"; adapterSeq[na++] = "AATGATACGGCGACCACCGACAGGTTCAGAGTTCTACAGTCCGA"; adapterNam[na] = "Illumina NlaIII expression Sequencing Primer"; adapterSeq[na++] = "CCGACAGGTTCAGAGTTCTACAGTCCGACATG"; adapterNam[na] = "Illumina Small RNA Adapter 1"; adapterSeq[na++] = "GTTCAGAGTTCTACAGTCCGACGATC"; adapterNam[na] = "Illumina Small RNA Adapter 2"; adapterSeq[na++] = "TCGTATGCCGTCTTCTGCTTGT"; adapterNam[na] = "Illumina Small RNA RT Primer"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina Small RNA PCR Primer 1"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina Small RNA PCR Primer 2"; adapterSeq[na++] = "AATGATACGGCGACCACCGACAGGTTCAGAGTTCTACAGTCCGA"; adapterNam[na] = "Illumina Small RNA Sequencing Primer"; adapterSeq[na++] = "CGACAGGTTCAGAGTTCTACAGTCCGACGATC"; adapterNam[na] = "Illumina Multiplexing Adapter 1"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCT"; adapterNam[na] = "Illumina Multiplexing Adapter 2"; adapterSeq[na++] = "ACACTCTTTCCCTACACGACGCTCTTCCGATCT"; adapterNam[na] = "Illumina Multiplexing PCR Primer 1.01"; adapterSeq[na++] = "AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTCCGATCT"; adapterNam[na] = "Illumina Multiplexing PCR Primer 2.01"; adapterSeq[na++] = "GTGACTGGAGTTCAGACGTGTGCTCTTCCGATCT"; adapterNam[na] = "Illumina Multiplexing Read1 Sequencing Primer"; adapterSeq[na++] = "ACACTCTTTCCCTACACGACGCTCTTCCGATCT"; adapterNam[na] = "Illumina Multiplexing Index Sequencing Primer"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCAC"; adapterNam[na] = "Illumina Multiplexing Read2 Sequencing Primer"; adapterSeq[na++] = "GTGACTGGAGTTCAGACGTGTGCTCTTCCGATCT"; adapterNam[na] = "Illumina PCR Primer Index 1"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCGTGATGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 2"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATACATCGGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 3"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGCCTAAGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 4"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTGGTCAGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 5"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCACTGTGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 6"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATATTGGCGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 7"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGATCTGGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 8"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTCAAGTGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 9"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCTGATCGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 10"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATAAGCTAGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 11"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGTAGCCGTGACTGGAGTTC"; adapterNam[na] = "Illumina PCR Primer Index 12"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTACAAGGTGACTGGAGTTC"; adapterNam[na] = "Illumina DpnII Gex Adapter 1"; adapterSeq[na++] = "GATCGTCGGACTGTAGAACTCTGAAC"; adapterNam[na] = "Illumina DpnII Gex Adapter 1.01"; adapterSeq[na++] = "ACAGGTTCAGAGTTCTACAGTCCGAC"; adapterNam[na] = "Illumina DpnII Gex Adapter 2"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina DpnII Gex Adapter 2.01"; adapterSeq[na++] = "TCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "Illumina DpnII Gex PCR Primer 1"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina DpnII Gex PCR Primer 2"; adapterSeq[na++] = "AATGATACGGCGACCACCGACAGGTTCAGAGTTCTACAGTCCGA"; adapterNam[na] = "Illumina DpnII Gex Sequencing Primer"; adapterSeq[na++] = "CGACAGGTTCAGAGTTCTACAGTCCGACGATC"; adapterNam[na] = "Illumina NlaIII Gex Adapter 1.01"; adapterSeq[na++] = "TCGGACTGTAGAACTCTGAAC"; adapterNam[na] = "Illumina NlaIII Gex Adapter 1.02"; adapterSeq[na++] = "ACAGGTTCAGAGTTCTACAGTCCGACATG"; adapterNam[na] = "Illumina NlaIII Gex Adapter 2.01"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina NlaIII Gex Adapter 2.02"; adapterSeq[na++] = "TCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "Illumina NlaIII Gex PCR Primer 1"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina NlaIII Gex PCR Primer 2"; adapterSeq[na++] = "AATGATACGGCGACCACCGACAGGTTCAGAGTTCTACAGTCCGA"; adapterNam[na] = "Illumina NlaIII Gex Sequencing Primer"; adapterSeq[na++] = "CCGACAGGTTCAGAGTTCTACAGTCCGACATG"; adapterNam[na] = "Illumina Small RNA RT Primer"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina 5p RNA Adapter"; adapterSeq[na++] = "GTTCAGAGTTCTACAGTCCGACGATC"; adapterNam[na] = "Illumina RNA Adapter1"; adapterSeq[na++] = "TCGTATGCCGTCTTCTGCTTGT"; adapterNam[na] = "Illumina Small RNA 3p Adapter 1"; adapterSeq[na++] = "ATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "Illumina Small RNA PCR Primer 1"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGA"; adapterNam[na] = "Illumina Small RNA PCR Primer 2"; adapterSeq[na++] = "AATGATACGGCGACCACCGACAGGTTCAGAGTTCTACAGTCCGA"; adapterNam[na] = "Illumina Small RNA Sequencing Primer"; adapterSeq[na++] = "CGACAGGTTCAGAGTTCTACAGTCCGACGATC"; adapterNam[na] = "TruSeq Universal Adapter"; adapterSeq[na++] = "AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTCCGATCT"; adapterNam[na] = "TruSeq Adapter, Index 1"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACATCACGATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 2"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACCGATGTATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 3"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACTTAGGCATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 4"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACTGACCAATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 5"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACACAGTGATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 6"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACGCCAATATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 7"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACCAGATCATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 8"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACACTTGAATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 9"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACGATCAGATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 10"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACTAGCTTATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 11"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACGGCTACATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "TruSeq Adapter, Index 12"; adapterSeq[na++] = "GATCGGAAGAGCACACGTCTGAACTCCAGTCACCTTGTAATCTCGTATGCCGTCTTCTGCTTG"; adapterNam[na] = "Illumina RNA RT Primer"; adapterSeq[na++] = "GCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "Illumina RNA PCR Primer"; adapterSeq[na++] = "AATGATACGGCGACCACCGAGATCTACACGTTCAGAGTTCTACAGTCCGA"; adapterNam[na] = "RNA PCR Primer, Index 1"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCGTGATGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 2"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATACATCGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 3"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGCCTAAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 4"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTGGTCAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 5"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCACTGTGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 6"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATATTGGCGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 7"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGATCTGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 8"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTCAAGTGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 9"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCTGATCGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 10"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATAAGCTAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 11"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGTAGCCGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 12"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTACAAGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 13"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTTGACTGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 14"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGGAACTGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 15"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTGACATGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 16"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGGACGGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 17"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCTCTACGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 18"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGCGGACGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 19"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTTTCACGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 20"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGGCCACGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 21"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCGAAACGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 22"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCGTACGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 23"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCCACTCGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 24"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGCTACCGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 25"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATATCAGTGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 26"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGCTCATGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 27"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATAGGAATGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 28"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCTTTTGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 29"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTAGTTGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 30"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCCGGTGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 31"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATATCGTGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 32"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTGAGTGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 33"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCGCCTGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 34"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGCCATGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 35"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATAAAATGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 36"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTGTTGGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 37"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATATTCCGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 38"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATAGCTAGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 39"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGTATAGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 40"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTCTGAGGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 41"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGTCGTCGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 42"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCGATTAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 43"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGCTGTAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 44"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATATTATAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 45"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATGAATGAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 46"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTCGGGAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 47"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATCTTCGAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; adapterNam[na] = "RNA PCR Primer, Index 48"; adapterSeq[na++] = "CAAGCAGAAGACGGCATACGAGATTGCCGAGTGACTGGAGTTCCTTGGCACCCGAGAATTCCA"; // Cre-Lox from Van Nieuwerburgh, et al. "Illumina mate-paired DNA sequencing-library preparation // using Cre-Lox recombination", Nucleic Acids Research, 2012, Vol 40, No. 3 adapterNam[na] = "CreLox 5'3'"; adapterSeq[na++] = "CGTAATAACTTCGTATAGCATACATTATACGAAGTTATACGA"; adapterNam[na] = "CreLox 3'5'"; adapterSeq[na++] = "GCATTATTGAAGCATATCGTATGTAATATGCTTCAATATGCT"; } if (0) { adapterNam[na] = "ABI Dynabead EcoP Oligo"; adapterSeq[na++] = "CTGATCTAGAGGTACCGGATCCCAGCAGT"; adapterNam[na] = "ABI Solid3 Adapter A"; adapterSeq[na++] = "CTGCCCCGGGTTCCTCATTCTCTCAGCAGCATG"; adapterNam[na] = "ABI Solid3 Adapter B"; adapterSeq[na++] = "CCACTACGCCTCCGCTTTCCTCTCTATGGGCAGTCGGTGAT"; adapterNam[na] = "ABI Solid3 5' AMP Primer"; adapterSeq[na++] = "CCACTACGCCTCCGCTTTCCTCTCTATG"; adapterNam[na] = "ABI Solid3 3' AMP Primer"; adapterSeq[na++] = "CTGCCCCGGGTTCCTCATTCT"; adapterNam[na] = "ABI Solid3 EF1 alpha Sense Primer"; adapterSeq[na++] = "CATGTGTGTTGAGAGCTTC"; adapterNam[na] = "ABI Solid3 EF1 alpha Antisense Primer"; adapterSeq[na++] = "GAAAACCAAAGTGGTCCAC"; adapterNam[na] = "ABI Solid3 GAPDH Forward Primer"; adapterSeq[na++] = "TTAGCACCCCTGGCCAAGG"; adapterNam[na] = "ABI Solid3 GAPDH Reverse Primer"; adapterSeq[na++] = "CTTACTCCTTGGAGGCCATG"; } if (adap454) { adapterNam[na] = "454 FLX Linker"; adapterSeq[na++] = "GTTGGAACCGAAAGGGTTTGAATTCAAACCCTTTCGGTTCCAAC"; adapterNam[na] = "454 Titanium Linker"; adapterSeq[na++] = "TCGTATAACTTCGTATAATGTATGCTATACGAAGTTATTACG"; adapterNam[na] = "454 AdaptorA"; adapterSeq[na++] = "CTGAGACAGGGAGGGAACAGATGGGACACGCAGGGATGAGATGG"; adapterNam[na] = "454 AdaptorB"; adapterSeq[na++] = "CTGAGACACGCAACAGGGGATAGGCAAGGCACACAGGGGATAGG"; } assert(na < 255); // Build a new sequence, from every adapter above, then every pair of adapter, F-F, F-R, R-F, R-R. // // For the pairs, we only care about the junction, so can limit to the two kmers on either side. uint32 adapterListMax = 16 * 1024 * 1024; char *adapterList = new char [adapterListMax]; char *adapterEnd = adapterList; memset(adapterList, 0, sizeof(char) * 16 * 1024 * 1024); int32 ladapter[256]; memset(ladapter, 0, sizeof(int32) * 256); char fadapterA[256]; char radapterA[256]; char fadapterB[256]; char radapterB[256]; for (uint32 a=0; agkStore_enableClearRange(AS_READ_CLEAR_OBTINITIAL); gkpStore->gkStore_enableClearRange(AS_READ_CLEAR_TNT); gkpStore->gkStore_metadataCaching(true); bool tntEnabled = false; // Over every result file, read each line, change the clear range. gkFragment gkf; mertrimResult res; fgets(resultFileName, FILENAME_MAX, listFile); while (!feof(listFile)) { chomp(resultFileName); errno = 0; FILE *resultFile = fopen(resultFileName, "r"); if (errno) fprintf(stderr, "Failed to open result file '%s' for reading: %s\n", resultFileName, strerror(errno)), exit(1); while (res.readResult(resultFile)) { gkpStore->gkStore_getFragment(res.readIID, &gkf, GKFRAGMENT_INF); if (res.chimer) gkf.gkFragment_setClearRegion(res.chmBgn, res.chmEnd, AS_READ_CLEAR_TNT); gkf.gkFragment_setClearRegion(res.clrBgn, res.clrEnd, AS_READ_CLEAR_OBTINITIAL); if (res.deleted) gkpStore->gkStore_delFragment(res.readIID); else gkpStore->gkStore_setFragment(&gkf); res.print(logFile); } fclose(resultFile); fgets(resultFileName, FILENAME_MAX, listFile); } fclose(listFile); fclose(logFile); gkpStore->gkStore_close(); } void merTrimShow(char *gkpStoreName, char *resultName) { errno = 0; FILE *resultFile = fopen(resultName, "r"); if (errno) fprintf(stderr, "Failed to open result file '%s' for reading: %s\n", resultName, strerror(errno)), exit(1); mertrimResult res; while (res.readResult(resultFile)) res.print(stdout); } int main(int argc, char **argv) { char *gkpStoreName = NULL; char *listName = NULL; char *logName = NULL; char *resultName = NULL; int arg = 1; int err = 0; while (arg < argc) { if (strcmp(argv[arg], "-g") == 0) { gkpStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-L") == 0) { listName = argv[++arg]; } else if (strcmp(argv[arg], "-l") == 0) { logName = argv[++arg]; } else if (strcmp(argv[arg], "-d") == 0) { resultName = argv[++arg]; } else { err++; } arg++; } if (((gkpStoreName == NULL) && (listName != NULL)) || ((gkpStoreName != NULL) && (listName == NULL))) err++; if ((listName == NULL) && (resultName == NULL)) err++; if ((listName != NULL) && (resultName != NULL)) err++; if (err) { fprintf(stderr, "usage: %s -L merTrimOutputList -g gkpStore [-l output.log]\n", argv[0]); fprintf(stderr, " %s -d merTrimOutput\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " The first form will read a list of merTrim output names from\n"); fprintf(stderr, " merTrimOuptutList, and apply the results to gkpStore.\n"); fprintf(stderr, "\n"); fprintf(stderr, " The second form will read a single merTrimOutput file and decode\n"); fprintf(stderr, " the results to stdout.\n"); fprintf(stderr, "\n"); fprintf(stderr, "\n"); if (((gkpStoreName == NULL) && (listName != NULL)) || ((gkpStoreName != NULL) && (listName == NULL))) fprintf(stderr, "ERROR: First form needs both -L and -g.\n"); if (gkpStoreName == NULL) fprintf(stderr, "ERROR: No gatekeeper store supplied with -g.\n"); if ((listName == NULL) && (resultName == NULL)) fprintf(stderr, "ERROR: Exactly one of -L and -d must be specified.\n"); if ((listName != NULL) && (resultName != NULL)) fprintf(stderr, "ERROR: Only one of -L and -d can be specified.\n"); fprintf(stderr, "\n"); exit(1); } if (listName) merTrimApply(gkpStoreName, listName, logName); if (resultName) merTrimShow(gkpStoreName, resultName); return(0); } canu-1.3/src/merTrim/merTrimResult.H000066400000000000000000000046731272335422300174550ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_MER/merTrimResult.H * * Modifications by: * * Brian P. Walenz from 2011-AUG-22 to 2013-AUG-01 * are Copyright 2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include "AS_global.H" #ifndef MERTRIMRESULT_H #define MERTRIMRESULT_H class mertrimResult { public: mertrimResult() { readIID = 0; deleted = 0; clrBgn = 0; clrEnd = 0; chimer = 0; chmBgn = 0; chmEnd = 0; }; void print(FILE *F) { if (F == NULL) return; if (chimer) fprintf(F, F_IID"\t"F_U32"\t"F_U32"\tchimer\t"F_U32"\t"F_U32"%s\n", readIID, clrBgn, clrEnd, chmBgn, chmEnd, (deleted) ? "\tdeleted" : ""); else fprintf(F, F_IID"\t"F_U32"\t"F_U32"%s\n", readIID, clrBgn, clrEnd, (deleted) ? "\tdeleted" : ""); }; void writeResult(FILE *W) { if (W == NULL) return; AS_UTL_safeWrite(W, this, "merTrimResult", sizeof(mertrimResult), 1); }; bool readResult(FILE *R) { if (R == NULL) return(false); if (!feof(R)) AS_UTL_safeRead(R, this, "merTrimResult", sizeof(mertrimResult), 1); return(feof(R) == false); }; AS_IID readIID; uint32 deleted; uint32 clrBgn; uint32 clrEnd; uint32 chimer; uint32 chmBgn; uint32 chmEnd; }; #endif // MERTRIMRESULT_H canu-1.3/src/mercy/000077500000000000000000000000001272335422300142145ustar00rootroot00000000000000canu-1.3/src/mercy/mercy-regions.C000066400000000000000000000256661272335422300171220ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_MER/mercy-regions.C * * Modifications by: * * Brian P. Walenz from 2007-MAR-19 to 2013-AUG-01 * are Copyright 2007-2008,2010-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-09 to 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include #include "AS_global.H" #include "splitToWords.H" #include "intervalList.H" // This reads the assembly frgctg, varctg and merQC badmers, computes // the number and location of bad-mer, bad-var regions, and their // depth, in contig space. // // File paths are hardcoded. // This code ONLY works on 64-bit hardware, but it's easy to fix. using namespace std; #include void readDepth(char *depthname, map*> &lowCoverage) { char line[1024] = {0}; map*> ILs; fprintf(stderr, "Reading depth from '%s'\n", depthname); errno = 0; FILE *F = fopen(depthname, "r"); if (errno) fprintf(stderr, "failed to open '%s': %s\n", depthname, strerror(errno)), exit(1); uint32 i=0; fgets(line, 1024, F); while (!feof(F)) { splitToWords W(line); uint64 uid = strtoul(W[1], 0L, 10); uint32 beg = strtoul(W[2], 0L, 10); uint32 end = strtoul(W[3], 0L, 10); if (beg > end) fprintf(stderr, "ERROR: l="F_U32" h="F_U32"\n", beg, end); if (ILs[uid] == 0L) ILs[uid] = new intervalList; ILs[uid]->add(beg, end - beg); i++; fgets(line, 1024, F); } fclose(F); fprintf(stderr, " "F_U32" lines.\n", i); map*>::iterator it = ILs.begin(); map*>::iterator ed = ILs.end(); while (it != ed) { lowCoverage[it->first] = new intervalList(*it->second); delete it->second; it->second = 0L; it++; } } void readVariation(char *depthname, map*> &variation) { char line[1024 * 1024] = {0}; fprintf(stderr, "Reading variation from '%s'\n", depthname); errno = 0; FILE *F = fopen(depthname, "r"); if (errno) fprintf(stderr, "failed to open '%s': %s\n", depthname, strerror(errno)), exit(1); uint32 i=0; fgets(line, 1024 * 1024, F); while (!feof(F)) { splitToWords W(line); uint64 uid = strtoul(W[1], 0L, 10); uint32 beg = strtoul(W[2], 0L, 10); uint32 end = strtoul(W[3], 0L, 10); if (variation[uid] == 0L) variation[uid] = new intervalList; variation[uid]->add(beg, end - beg); i++; fgets(line, 1024 * 1024, F); } fclose(F); fprintf(stderr, " "F_U32" lines.\n", i); } void readBadMers(char *depthname, map*> &badMers) { char line[1024] = {0}; fprintf(stderr, "Reading badMers from '%s'\n", depthname); errno = 0; FILE *F = fopen(depthname, "r"); if (errno) fprintf(stderr, "failed to open '%s': %s\n", depthname, strerror(errno)), exit(1); uint32 i=0; fgets(line, 1024, F); while (!feof(F)) { splitToWords W(line); // Change every non-digit to a space in the first word. for (uint32 z=strlen(W[0])-1; z--; ) if (!isdigit(W[0][z])) W[0][z] = ' '; uint64 uid = strtoul(W[0], 0L, 10); uint32 beg = strtoul(W[3], 0L, 10); uint32 end = strtoul(W[4], 0L, 10); if (badMers[uid] == 0L) badMers[uid] = new intervalList; badMers[uid]->add(beg, end - beg); i++; fgets(line, 1024, F); } fclose(F); fprintf(stderr, " "F_U32" lines.\n", i); } int main(int argc, char **argv) { map*> badMers; map*> variation; map*> lowCoverage; bool showDepthIntersect = false; bool showVariantIntersect = false; bool showVarDepthIntersect = false; argc = AS_configure(argc, argv); int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-D") == 0) { } else if (strcmp(argv[arg], "-pd") == 0) { showDepthIntersect = true; } else if (strcmp(argv[arg], "-pv") == 0) { showVariantIntersect = true; } else if (strcmp(argv[arg], "-pvd") == 0) { showVarDepthIntersect = true; } else { fprintf(stderr, "usage: %s [-D debugfile] [-pd] [-pv] [-pvd]\n", argv[0]); fprintf(stderr, " -pd print bad mers regions isect depth\n"); fprintf(stderr, " -pv print bad mers regions isect variants\n"); fprintf(stderr, " -pvd print bad mers regions isect both variants and depth\n"); exit(1); } arg++; } #if 1 // HuRef6, in the assembly directory. // readDepth ("/project/huref6/assembly/h6/9-terminator/h6.posmap.frgctg", lowCoverage); readVariation("/project/huref6/assembly/h6/9-terminator/h6.posmap.varctg", variation); readBadMers ("/project/huref6/assembly/h6-mer-validation/h6-ms22-allfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers", badMers); #endif #if 0 // HuRef6, ws=25, in the assembly directory. // readDepth ("/project/huref6/assembly/h6/9-terminator-ws25/h6.posmap.frgctg", lowCoverage); readVariation("/project/huref6/assembly/h6/9-terminator-ws25/h6.posmap.varctg", variation); readBadMers ("/project/huref6/assembly/h6-mer-validation/h6-version4-ws25/h6-ms22-allfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers", badMers); #endif #if 0 // Our scratch huref // readDepth ("/project/huref6/redo_consensus-gennady/mer-validation/h6tmp.posmap.frgctg", lowCoverage); readVariation("/project/huref6/redo_consensus-gennady/mer-validation/h6tmp.posmap.varctg", variation); readBadMers ("/project/huref6/redo_consensus-gennady/mer-validation/h6tmp-ms22-allfrags-allcontigs.badmers.0.singlecontig.zerofrag.badmers", badMers); #endif uint32 badBegDepth[1024] = {0}; uint32 badEndDepth[1024] = {0}; uint32 badDepth[32][32]; for (uint32 i=0; i<32; i++) for (uint32 j=0; j<32; j++) badDepth[i][j] = 0; map*>::iterator it = badMers.begin(); map*>::iterator ed = badMers.end(); while (it != ed) { uint64 uid = it->first; intervalList *Iv = variation[uid]; intervalList *Ib = badMers[uid]; intervalList *Ii = 0L; intervalList *Id = lowCoverage[uid]; if (Iv) Iv->merge(); if (Ib) Ib->merge(); if (Iv && Ib) { Ii = new intervalList; Ii->intersect(*Iv, *Ib); } if (Ii) { uint32 ii = 0; uint32 id = 0; while ((ii < Ii->numberOfIntervals()) && (id < Id->numberOfIntervals())) { // We want to count the number of times a badmer region // begins/ends in some depth. //fprintf(stderr, "testing beg "F_U32" "F_U32" -- "F_U32" "F_U32"\n", // Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id)); uint32 beg = 0; uint32 end = 0; // Low points are not allowed to be equal to high points, skip to the next while ((id < Id->numberOfIntervals()) && (Id->hi(id) <= Ii->lo(ii))) { id++; //fprintf(stderr, "testing beg (m) "F_U32" "F_U32" -- "F_U32" "F_U32"\n", // Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id)); } if (id < Id->numberOfIntervals()) { uint32 lo = Id->lo(id); uint32 hi = Id->hi(id); // Low points are not allowed to be equal to high points. if ((lo <= Ii->lo(ii)) && (Ii->lo(ii) < hi)) { beg = Id->depth(id); } else { fprintf(stderr, "failed to find begin "F_U32" "F_U32" -- "F_U32" "F_U32" "F_U32"\n", Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id), Id->depth(id)); if (id > 0) fprintf(stderr, " "F_U32" "F_U32" -- "F_U32" "F_U32" "F_U32"\n", Ii->lo(ii), Ii->hi(ii), Id->lo(id-1), Id->hi(id-1), Id->depth(id-1)); //exit(1); } } //fprintf(stderr, "testing end "F_U64" "F_U64" -- "F_U64" "F_U64"\n", // Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id)); // High points can be equal. while ((id < Id->numberOfIntervals()) && (Id->hi(id) < Ii->hi(ii))) { id++; //fprintf(stderr, "testing end (m) "F_U64" "F_U64" -- "F_U64" "F_U64"\n", // Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id)); } if (id < Id->numberOfIntervals()) { uint32 lo = Id->lo(id); uint32 hi = Id->hi(id); // High points aren't allowed to be equal to lo, but can be equal to hi. if ((lo < Ii->hi(ii)) && (Ii->hi(ii) <= hi)) { end = Id->depth(id); } else { fprintf(stderr, "failed to find end "F_U32" "F_U32" -- "F_U32" "F_U32" "F_U32"\n", Ii->lo(ii), Ii->hi(ii), Id->lo(id), Id->hi(id), Id->depth(id)); if (id > 0) fprintf(stderr, " "F_U32" "F_U32" -- "F_U32" "F_U32" "F_U32"\n", Ii->lo(ii), Ii->hi(ii), Id->lo(id-1), Id->hi(id-1), Id->depth(id-1)); //exit(1); } } badBegDepth[beg]++; badEndDepth[end]++; fprintf(stdout, F_U64"\t"F_U32"\t"F_U32"\tdepth="F_U32","F_U32"\n", uid, Ii->lo(ii), Ii->hi(ii), beg, end); if ((beg < 32) && (end < 32)) badDepth[beg][end]++; ii++; } } it++; } uint32 bb = 0; uint32 be = 0; for (uint32 x=0; x<32; x++) { fprintf(stdout, F_U32"\t"F_U32"\t"F_U32"\n", x, badBegDepth[x], badEndDepth[x]); bb += badBegDepth[x]; be += badEndDepth[x]; } fprintf(stdout, "total\t"F_U32"\t"F_U32"\n", bb, be); for (uint32 i=0; i<30; i++) { for (uint32 j=0; j<30; j++) fprintf(stdout, "%5u", badDepth[i][j]); fprintf(stdout, "\n"); } return(0); } canu-1.3/src/mercy/mercy.C000066400000000000000000000337061272335422300154500ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_MER/mercy.C * * Modifications by: * * Brian P. Walenz from 2007-MAR-19 to 2014-APR-11 * are Copyright 2007-2009,2011,2013-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include "AS_global.H" #include "libmeryl.H" // The categories depend on the type of input (fragments or contigs): // // 0 -- no count, mer not present // 1 -- single copy // 2 -- 2 -> 10 copies (contigs) -- 2 -> 2mode copies (frags) // 3 -- 11 -> 100 copies (contigs) -- -> 10mode copies (frags) // 4 -- 101+ copies (contigs) -- -> 100mode copies (frags) // 5 -- -> infinity copies (frags) // You'll also need to modify compare() and output() if you change this. #define NUMCATEGORIES 6 // The output files are global for convenience. Otherwise, we'd be passing // them to compare() for every single mer. // bool dumpFlag = false; FILE *dumpSCZF = 0L; FILE *dumpMCZF = 0L; FILE *dumpMCSF = 0L; FILE *dumpMCMF = 0L; char merstring[1024]; uint32 findMode(char *name) { merylStreamReader *M = new merylStreamReader(name); uint32 *H = new uint32 [16384]; fprintf(stderr, "Finding mode of '%s'\n", name); for (uint32 i=0; i<16384; i++) H[i] = 0; while (M->validMer()) { if (M->theCount() < 16384) H[M->theCount()]++; M->nextMer(); } uint32 mi = 16; for (uint32 i=mi; i<16384; i++) if (H[i] > H[mi]) mi = i; fprintf(stderr, "Mode of '%s' is "F_U32"\n", name, mi); return(mi); } void compare(merylStreamReader *F, merylStreamReader *C, kMer &minmer, uint32 mode, uint32 R[NUMCATEGORIES][NUMCATEGORIES]) { uint32 Ftype = 0; uint32 Ctype = 0; kMer Fmer = F->theFMer(); kMer Cmer = C->theFMer(); uint32 Fcnt = F->theCount(); uint32 Ccnt = C->theCount(); if (Fcnt == 0) Ftype = 0; else if (Fcnt == 1) Ftype = 1; else if (Fcnt <= 2*mode) Ftype = 2; else if (Fcnt <= 10*mode) Ftype = 3; else if (Fcnt <= 100*mode) Ftype = 4; else Ftype = 5; if (Ccnt == 0) Ctype = 0; else if (Ccnt == 1) Ctype = 1; else if (Ccnt <= 10) Ctype = 2; else if (Ccnt <= 100) Ctype = 3; else Ctype = 4; // If the mer isn't valid, we hit the end of the file, and the mer // thus (obviously) isn't in the file. // if (F->validMer() == false) Ftype = 0; if (C->validMer() == false) Ctype = 0; // If either type is 0, we're done, but only increment the count if // this mer is the minmer. // if ((Ftype == 0) || (Ctype == 0)) { if (((Ftype == 0) && (Cmer == minmer)) || ((Ctype == 0) && (Fmer == minmer))) { R[Ftype][Ctype]++; // Save the mer if it's in contigs, but not fragments. if (dumpFlag) if (Ftype == 0) if (Ctype == 1) fprintf(dumpSCZF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring)); else fprintf(dumpMCZF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring)); } return; } // If the mers don't agree, we're also done. If either is the // minmer, note that we saw it. // if (Fmer != Cmer) { if (Fmer == minmer) R[Ftype][0]++; if (Cmer == minmer) { R[0][Ctype]++; // Again, save the mer since it's in contigs, but not fragments. if (dumpFlag) if (Ctype == 1) fprintf(dumpSCZF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring)); else fprintf(dumpMCZF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring)); } return; } // If we're not the minmer, we're done. if (Fmer != minmer) return; // Otherwise, the mers are in both inputs R[Ftype][Ctype]++; // Save the mer if it's in contigs "more" than if in fragments. if (dumpFlag) { if (Ftype < Ctype) if (Ctype == 2) fprintf(dumpMCSF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring)); else fprintf(dumpMCMF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring)); if ((Ftype == 0) && (Ctype == 1)) fprintf(dumpSCZF, ">"F_U32"\n%s\n", Ccnt, Cmer.merToString(merstring)); } } void output(char *title, uint32 mode, uint32 R[NUMCATEGORIES][NUMCATEGORIES]) { fprintf(stdout, "\n\n%s\n", title); fprintf(stdout, "(frags) | zero | one | <= 10 | <= 100 | <= inf | (contigs)\n"); for (uint32 i=0; i<6; i++) { switch (i) { case 0: fprintf(stdout, "zero "); break; case 1: fprintf(stdout, "one "); break; case 2: fprintf(stdout, "<= 2mode "); break; case 3: fprintf(stdout, "<= 10mode "); break; case 4: fprintf(stdout, "<= 100mode "); break; case 5: fprintf(stdout, "<= inf "); break; default: fprintf(stdout, "????????? "); break; } for (uint32 j=0; j<5; j++) fprintf(stdout, "%12"F_U32P, R[i][j]); fprintf(stdout, "\n"); } } int main(int argc, char **argv) { merylStreamReader *AF = 0L; merylStreamReader *TF = 0L; merylStreamReader *AC = 0L; merylStreamReader *DC = 0L; merylStreamReader *CO = 0L; uint32 AFmode = 0; uint32 TFmode = 0; char dumpSCZFname[1024] = {0}; // single contig, zero frags char dumpMCZFname[1024] = {0}; // low contig, zero frags char dumpMCSFname[1024] = {0}; // medium contig, low frags char dumpMCMFname[1024] = {0}; // everything else, contig > frags bool beVerbose = false; argc = AS_configure(argc, argv); int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-af") == 0) { // All frags ++arg; AFmode = findMode(argv[arg]); AF = new merylStreamReader(argv[arg]); AF->nextMer(); } else if (strcmp(argv[arg], "-tf") == 0) { // Trimmed frags ++arg; TFmode = findMode(argv[arg]); TF = new merylStreamReader(argv[arg]); TF->nextMer(); } else if (strcmp(argv[arg], "-ac") == 0) { // All contigs AC = new merylStreamReader(argv[++arg]); AC->nextMer(); } else if (strcmp(argv[arg], "-dc") == 0) { // Degenerate contigs DC = new merylStreamReader(argv[++arg]); DC->nextMer(); } else if (strcmp(argv[arg], "-co") == 0) { // Contigs CO = new merylStreamReader(argv[++arg]); CO->nextMer(); } else if (strcmp(argv[arg], "-dump") == 0) { arg++; dumpFlag = true; sprintf(dumpSCZFname, "%s.0.singlecontig.zerofrag.fasta", argv[arg]); sprintf(dumpMCZFname, "%s.1.multiplecontig.zerofrag.fasta", argv[arg]); sprintf(dumpMCSFname, "%s.2.multiplecontig.lowfrag.fasta", argv[arg]); sprintf(dumpMCMFname, "%s.3.multiplecontig.multiplefrag.fasta", argv[arg]); } else if (strcmp(argv[arg], "-v") == 0) { beVerbose = true; } else { fprintf(stderr, "unknown option '%s'\n", argv[arg]); } arg++; } if ((AF == 0L) && (TF == 0L) && (AC == 0L) && (DC == 0L) && (CO == 0L)) { fprintf(stderr, "usage: %s [opts] [-v] [-dump prefix]\n", argv[0]); fprintf(stderr, "At least one fragcounts and one contigcounts are needed.\n"); fprintf(stderr, " -af | -tf fragcounts\n"); fprintf(stderr, " -ac | -dc | -co contigcounts \n"); fprintf(stderr, "Dumping is probably only useful with exactly one frag and\n"); fprintf(stderr, "one contig, but I'll let you do it with any number.\n"); exit(1); } if ((AF == 0L) && (TF == 0L)) { fprintf(stderr, "ERROR - need at least one of -af, -tf\n"); exit(1); } if ((AC == 0L) && (DC == 0L) && (CO == 0L)) { fprintf(stderr, "ERROR - need at least one of -ac, -dc, -co\n"); exit(1); } // Check mersizes. // uint32 merSize = 0; uint32 ms[5] = { 0 }; if (AF) merSize = ms[0] = AF->merSize(); if (TF) merSize = ms[1] = TF->merSize(); if (AC) merSize = ms[2] = AC->merSize(); if (DC) merSize = ms[3] = DC->merSize(); if (CO) merSize = ms[4] = CO->merSize(); bool differ = false; if ((ms[0] > 0) && (ms[0] != merSize)) differ = true; if ((ms[1] > 0) && (ms[1] != merSize)) differ = true; if ((ms[2] > 0) && (ms[2] != merSize)) differ = true; if ((ms[3] > 0) && (ms[3] != merSize)) differ = true; if ((ms[4] > 0) && (ms[4] != merSize)) differ = true; if (differ) { fprintf(stderr, "error: mer size differ.\n"); fprintf(stderr, " AF - "F_U32"\n", ms[0]); fprintf(stderr, " TF - "F_U32"\n", ms[1]); fprintf(stderr, " AC - "F_U32"\n", ms[2]); fprintf(stderr, " DC - "F_U32"\n", ms[3]); fprintf(stderr, " CO - "F_U32"\n", ms[4]); exit(1); } if (dumpFlag) { errno = 0; dumpSCZF = fopen(dumpSCZFname, "w"); dumpMCZF = fopen(dumpMCZFname, "w"); dumpMCSF = fopen(dumpMCSFname, "w"); dumpMCMF = fopen(dumpMCMFname, "w"); if (errno) fprintf(stderr, "Failed to open the dump files: %s\n", strerror(errno)), exit(1); } uint32 AFvsAC[NUMCATEGORIES][NUMCATEGORIES]; uint32 AFvsDC[NUMCATEGORIES][NUMCATEGORIES]; uint32 AFvsCO[NUMCATEGORIES][NUMCATEGORIES]; uint32 TFvsAC[NUMCATEGORIES][NUMCATEGORIES]; uint32 TFvsDC[NUMCATEGORIES][NUMCATEGORIES]; uint32 TFvsCO[NUMCATEGORIES][NUMCATEGORIES]; for (uint32 i=0; ivalidMer()) minmer = AF->theFMer(); if (TF && TF->validMer()) minmer = TF->theFMer(); if (AC && AC->validMer()) minmer = AC->theFMer(); if (DC && DC->validMer()) minmer = DC->theFMer(); if (CO && CO->validMer()) minmer = CO->theFMer(); speedCounter *C = new speedCounter(" Examining: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose); bool morestuff = true; while (morestuff) { // Find any mer in our set if (AF && AF->validMer()) minmer = AF->theFMer(); if (TF && TF->validMer()) minmer = TF->theFMer(); if (AC && AC->validMer()) minmer = AC->theFMer(); if (DC && DC->validMer()) minmer = DC->theFMer(); if (CO && CO->validMer()) minmer = CO->theFMer(); // Find the smallest mer in our set if (AF && AF->validMer() && (AF->theFMer() < minmer)) minmer = AF->theFMer(); if (TF && TF->validMer() && (TF->theFMer() < minmer)) minmer = TF->theFMer(); if (AC && AC->validMer() && (AC->theFMer() < minmer)) minmer = AC->theFMer(); if (DC && DC->validMer() && (DC->theFMer() < minmer)) minmer = DC->theFMer(); if (CO && CO->validMer() && (CO->theFMer() < minmer)) minmer = CO->theFMer(); // We need to do up to six comparisons here. if (AF && AC) compare(AF, AC, minmer, AFmode, AFvsAC); if (AF && DC) compare(AF, DC, minmer, AFmode, AFvsDC); if (AF && CO) compare(AF, CO, minmer, AFmode, AFvsCO); if (TF && AC) compare(TF, AC, minmer, TFmode, TFvsAC); if (TF && DC) compare(TF, DC, minmer, TFmode, TFvsDC); if (TF && CO) compare(TF, CO, minmer, TFmode, TFvsCO); C->tick(); #if 0 if (C->tick()) { char stringjunk[256]; fprintf(stderr, "\nMM %s\n", minmer.merToString(stringjunk)); if (AF) fprintf(stderr, "AF %s\n", AF->theFMer().merToString(stringjunk)); if (TF) fprintf(stderr, "TF %s\n", TF->theFMer().merToString(stringjunk)); if (AC) fprintf(stderr, "AC %s\n", AC->theFMer().merToString(stringjunk)); if (DC) fprintf(stderr, "DC %s\n", DC->theFMer().merToString(stringjunk)); if (CO) fprintf(stderr, "CO %s\n", CO->theFMer().merToString(stringjunk)); } #endif // Advance to the next mer, if we were just used morestuff = false; if ((AF) && (AF->theFMer() == minmer)) morestuff |= AF->nextMer(); if ((TF) && (TF->theFMer() == minmer)) morestuff |= TF->nextMer(); if ((AC) && (AC->theFMer() == minmer)) morestuff |= AC->nextMer(); if ((DC) && (DC->theFMer() == minmer)) morestuff |= DC->nextMer(); if ((CO) && (CO->theFMer() == minmer)) morestuff |= CO->nextMer(); } delete C; // output if ((AF) && (AC)) output("all frags vs all contigs", AFmode, AFvsAC); if ((AF) && (DC)) output("all frags vs deg. contigs", AFmode, AFvsDC); if ((AF) && (CO)) output("all frags vs non-deg. contigs", AFmode, AFvsCO); if ((TF) && (AC)) output("trimmed frags vs all contigs", TFmode, TFvsAC); if ((TF) && (DC)) output("trimmed frags vs deg. contigs", TFmode, TFvsDC); if ((TF) && (CO)) output("trimmed frags vs non-deg. contigs", TFmode, TFvsCO); delete AF; delete TF; delete AC; delete DC; delete CO; exit(0); } canu-1.3/src/mercy/mercy.sh000066400000000000000000000160241272335422300156720ustar00rootroot00000000000000#!/bin/sh # This file is part of Celera Assembler, a software program that # assembles whole-genome shotgun reads into contigs and scaffolds. # Copyright (C) 2006-2007, J. Craig Venter Institute # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received (LICENSE.txt) a copy of the GNU General Public # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Test if the mers in the consensus sequence are supported by mers in # the fragments. # If we count just the clear, we get a clearer (ha, ha) picture of # the assembly quality, while if we count all reads we get a picture # of trimming. # onlyClear=1 onlyReal=1 mem=8192 mem=16384 mem=24576 ms=22 binroot=/bioinfo/assembly/walenz/src/genomics asmMerQC=$binroot/meryl/asmMerQC mapMers=$binroot/meryl/mapMers dir=/scratch/drosnightly asm=willi dir=/project/huref6/redo-consensus_gennady asm=h6tmp dir=/project/huref6/assembly/h6 asm=h6 # Count mers in reads # if [ ! -e $asm-ms$ms-clr-frags.mcidx ] ; then bin/dumpFragStoreAsFasta -frg $dir/$asm.frgStore | \ meryl -B -C -m $ms -s - -o $asm-ms$ms-clr-frags -threads 4 -memory $mem -v fi if [ ! -e $asm-ms$ms-all-frags.mcidx ] ; then bin/dumpFragStoreAsFasta -allbases -allfrags -frg $dir/$asm.frgStore | \ meryl -B -C -m $ms -s - -o $asm-ms$ms-all-frags -threads 4 -memory $mem -v fi echo Finding contigs. if [ ! -e $asm.normalcontigs.fasta ] ; then bin/asmOutputContigsFasta < $dir/9-terminator/$asm.asm > $asm.normalcontigs.fasta & fi if [ ! -e $asm.degeneratecontigs.fasta ] ; then bin/asmOutputContigsFasta -D < $dir/9-terminator/$asm.asm > $asm.degeneratecontigs.fasta & fi if [ ! -e $asm.allcontigs.fasta ] ; then bin/asmOutputContigsFasta -d < $dir/9-terminator/$asm.asm > $asm.allcontigs.fasta & fi # Count mers in contigs # if [ ! -e $asm-ms$ms-normal-contigs.mcidx ] ; then meryl -B -C -m $ms -s $asm.normalcontigs.fasta -o $asm-ms$ms-normal-contigs -threads 4 -segments 4 -v & fi if [ ! -e $asm-ms$ms-degenerate-contigs.mcidx ] ; then meryl -B -C -m $ms -s $asm.degeneratecontigs.fasta -o $asm-ms$ms-degenerate-contigs -threads 4 -segments 4 -v & fi if [ ! -e $asm-ms$ms-all-contigs.mcidx ] ; then meryl -B -C -m $ms -s $asm.allcontigs.fasta -o $asm-ms$ms-all-contigs -threads 4 -segments 4 -v & fi if [ ! -e $asm-ms$ms.asmMerQC ] ; then $asmMerQC -af $asm-ms$ms-all-frags \ -tf $asm-ms$ms-clr-frags \ -co $asm-ms$ms-normal-contigs \ -ac $asm-ms$ms-all-contigs \ -dc $asm-ms$ms-degenerate-contigs \ > $asm-ms$ms.asmMerQC & fi echo Finding badmers. if [ ! -e $asm-ms$ms-allfrags-normalcontigs.badmers.asmMerQC ] ; then $asmMerQC -af $asm-ms$ms-all-frags \ -co $asm-ms$ms-normal-contigs \ -dump $asm-ms$ms-allfrags-normalcontigs.badmers \ > $asm-ms$ms-allfrags-normalcontigs.badmers.asmMerQC & fi if [ ! -e $asm-ms$ms-allfrags-allcontigs.badmers.asmMerQC ] ; then $asmMerQC -af $asm-ms$ms-all-frags \ -ac $asm-ms$ms-all-contigs \ -dump $asm-ms$ms-allfrags-allcontigs.badmers \ > $asm-ms$ms-allfrags-allcontigs.badmers.asmMerQC & fi if [ ! -e $asm-ms$ms-allfrags-degeneratecontigs.badmers.asmMerQC ] ; then $asmMerQC -af $asm-ms$ms-all-frags \ -dc $asm-ms$ms-degenerate-contigs \ -dump $asm-ms$ms-allfrags-degeneratecontigs.badmers \ > $asm-ms$ms-allfrags-degeneratecontigs.badmers.asmMerQC & fi if [ ! -e $asm-ms$ms-clrfrags-normalcontigs.badmers.asmMerQC ] ; then $asmMerQC -tf $asm-ms$ms-clr-frags \ -co $asm-ms$ms-normal-contigs \ -dump $asm-ms$ms-clrfrags-normalcontigs.badmers \ > $asm-ms$ms-clrfrags-normalcontigs.badmers.asmMerQC & fi if [ ! -e $asm-ms$ms-clrfrags-allcontigs.badmers.asmMerQC ] ; then $asmMerQC -tf $asm-ms$ms-clr-frags \ -ac $asm-ms$ms-all-contigs \ -dump $asm-ms$ms-clrfrags-allcontigs.badmers \ > $asm-ms$ms-clrfrags-allcontigs.badmers.asmMerQC & fi if [ ! -e $asm-ms$ms-clrfrags-degeneratecontigs.badmers.asmMerQC ] ; then $asmMerQC -tf $asm-ms$ms-clr-frags \ -dc $asm-ms$ms-degenerate-contigs \ -dump $asm-ms$ms-clrfrags-degeneratecontigs.badmers \ > $asm-ms$ms-clrfrags-degeneratecontigs.badmers.asmMerQC & fi echo Mapping. if [ ! -e $asm-ms$ms-allfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then $mapMers -m 22 \ -mers $asm-ms$ms-allfrags-normalcontigs.badmers.0.singlecontig.zerofrag.fasta \ -seq $asm.normalcontigs.fasta \ > $asm-ms$ms-allfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers & fi if [ ! -e $asm-ms$ms-allfrags-allcontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then $mapMers -m 22 \ -mers $asm-ms$ms-allfrags-allcontigs.badmers.0.singlecontig.zerofrag.fasta \ -seq $asm.allcontigs.fasta \ > $asm-ms$ms-allfrags-allcontigs.badmers.0.singlecontig.zerofrag.badmers & fi if [ ! -e $asm-ms$ms-allfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then $mapMers -m 22 \ -mers $asm-ms$ms-allfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.fasta \ -seq $asm.degeneratecontigs.fasta \ > $asm-ms$ms-allfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.badmers & fi if [ ! -e $asm-ms$ms-clrfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then $mapMers -m 22 \ -mers $asm-ms$ms-clrfrags-normalcontigs.badmers.0.singlecontig.zerofrag.fasta \ -seq $asm.normalcontigs.fasta \ > $asm-ms$ms-clrfrags-normalcontigs.badmers.0.singlecontig.zerofrag.badmers & fi if [ ! -e $asm-ms$ms-clrfrags-allcontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then $mapMers -m 22 \ -mers $asm-ms$ms-clrfrags-allcontigs.badmers.0.singlecontig.zerofrag.fasta \ -seq $asm.allcontigs.fasta \ > $asm-ms$ms-clrfrags-allcontigs.badmers.0.singlecontig.zerofrag.badmers & fi if [ ! -e $asm-ms$ms-clrfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.badmers ] ; then $mapMers -m 22 \ -mers $asm-ms$ms-clrfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.fasta \ -seq $asm.degeneratecontigs.fasta \ > $asm-ms$ms-clrfrags-degeneratecontigs.badmers.0.singlecontig.zerofrag.badmers & fi if [ ! -e $asm-ms$ms-allfrags-normalcontigs.badmers.5.all.badmers ] ; then cat $asm-ms$ms-allfrags-normalcontigs.badmers.[01].*.fasta > $asm-ms$ms-allfrags-normalcontigs.badmers.5.allzero.fasta $mapMers -m 22 \ -mers $asm-ms$ms-allfrags-normalcontigs.badmers.5.allzero.fasta \ -seq $asm.normalcontigs.fasta \ > $asm-ms$ms-allfrags-normalcontigs.badmers.5.allzero.badmers & fi date canu-1.3/src/meryl/000077500000000000000000000000001272335422300142255ustar00rootroot00000000000000canu-1.3/src/meryl/compare-counts.C000066400000000000000000000142101272335422300172660ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/compare-counts.C * * Modifications by: * * Brian P. Walenz from 2010-AUG-28 to 2014-APR-11 * are Copyright 2010,2012,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include #include "libmeryl.H" #if 0 void heatMap() { speedCounter *C = new speedCounter(" Examining: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1ffffff, false); #define MAXA 150 #define MAXB 150 double heatraw[MAXA][MAXB]; double heatsca[MAXA][MAXB]; for (uint32 i=0; inextMer(); B->nextMer(); while ((A->validMer()) || (B->validMer())) { kMer &a = A->theFMer(); kMer &b = B->theFMer(); uint32 ac = A->theCount(); uint32 bc = B->theCount(); if (ac >= MAXA) ac = MAXA-1; if (bc >= MAXB) bc = MAXB-1; if (A->validMer() == false) { ac = 0; heatraw[ac][bc]++; B->nextMer(); continue; } if (B->validMer() == false) { bc = 0; heatraw[ac][bc]++; A->nextMer(); continue; } if (a == b) { heatraw[ac][bc]++; A->nextMer(); B->nextMer(); } else if (a < b) { heatraw[ac][0]++; A->nextMer(); } else { heatraw[0][bc]++; B->nextMer(); } C->tick(); } delete C; delete A; delete B; // Scale each row to be between 0 and 1 #if 0 for (uint32 j=0; jmerSize(); #define HMAX 64 * 1024 uint32 *Htrue = new uint32 [HMAX]; uint32 *Hnoise = new uint32 [HMAX]; for (uint32 i=0; inextMer(); S->nextMer(); while ((T->validMer()) || (S->validMer())) { kMer &t = T->theFMer(); kMer &s = S->theFMer(); uint32 tc = T->theCount(); uint32 sc = S->theCount(); if (tc >= HMAX) tc = HMAX-1; if (sc >= HMAX) sc = HMAX-1; // If we're out of truth kmers, the sample is noise. if (T->validMer() == false) { Hnoise[sc]++; S->nextMer(); continue; } // If we're out of sample kmers, do nothing but go to the next truth kmer. if (S->validMer() == false) { T->nextMer(); continue; } // If the kmers are equal, this is a true kmer if (t == s) { Htrue[sc]++; T->nextMer(); S->nextMer(); } // If the truth kmer is the lesser, get the next truth. else if (t < s) { T->nextMer(); } // Else the sample kmer is smaller, add it to the noise pile, and get the next. else { Hnoise[sc]++; S->nextMer(); } } delete T; delete S; char outputName[FILENAME_MAX]; sprintf(outputName, "%s.gp", outputPrefix); FILE *outputGP = fopen(outputName, "w"); sprintf(outputName, "%s.dat", outputPrefix); FILE *outputDAT = fopen(outputName, "w"); fprintf(outputGP, "set terminal png\n"); fprintf(outputGP, "set output \"%s.png\"\n", outputPrefix); fprintf(outputGP, "set title \"%s true/false %d-mers\"\n", plotTitle, kmerSize); fprintf(outputGP, "set xlabel \"k-mer count\"\n"); fprintf(outputGP, "set ylabel \"number of kmers\"\n"); fprintf(outputGP, "plot [0:100] [0:1000000] \"%s.dat\" using 1:2 with lines title \"true\", \"%s.dat\" using 1:3 with lines title \"false\"\n", outputPrefix, outputPrefix); fclose(outputGP); for (uint32 i=0; inumberOfDistinctMers() - MF->numberOfUniqueMers(); uint64 totalUsefulAll = MF->numberOfTotalMers() - MF->numberOfUniqueMers(); uint64 distinct = 0; uint64 total = 0; uint32 Xcoverage = 8; fprintf(stderr, "distinct: "F_U64"\n", MF->numberOfDistinctMers()); fprintf(stderr, "unique: "F_U64"\n", MF->numberOfUniqueMers()); fprintf(stderr, "total: "F_U64"\n", MF->numberOfTotalMers()); // Pass 0: try to deduce the X coverage we have. The // pattern we should see in mer counts is an initial spike // for unique mers (these contain errors), then a drop into // a valley, and a bump at the X coverage. // // . // . ... // .. .......... // ................. // // If this pattern is not found, we fallback to the default // guess of 8x coverage. // uint32 i = 0; uint32 iX = 0; fprintf(stderr, "distinct: "F_U64"\n", MF->numberOfDistinctMers()); fprintf(stderr, "unique: "F_U64"\n", MF->numberOfUniqueMers()); fprintf(stderr, "total: "F_U64"\n", MF->numberOfTotalMers()); fprintf(stderr, "Xcoverage zero 1 0 "F_U64"\n", MF->histogram(1)); for (i=2; (i < MF->histogramLength()) && (MF->histogram(i-1) > MF->histogram(i)); i++) fprintf(stderr, "Xcoverage drop "F_U32" "F_U64" "F_U64"\n", i, MF->histogram(i-1), MF->histogram(i)); iX = i - 1; for (; i < MF->histogramLength(); i++) { if (MF->histogram(iX) < MF->histogram(i)) { fprintf(stderr, "Xcoverage incr "F_U32" "F_U64" "F_U64"\n", i, MF->histogram(iX), MF->histogram(i)); iX = i; } else { //fprintf(stderr, "Xcoverage drop "F_U32" "F_U64" "F_U64"\n", i, MF->histogram(iX), MF->histogram(i)); } } fprintf(stderr, "Guessed X coverage is "F_U32"\n", iX); Xcoverage = iX; // Pass 1: look for a reasonable limit, using %distinct and %total. // for (i=2; (i < MF->histogramLength()) && (maxCount == 0); i++) { distinct += MF->histogram(i); total += MF->histogram(i) * i; // If we cover 99% of all the distinct mers, that's reasonable. // if ((distinct / (double)totalUsefulDistinct) > 0.99) maxCount = i; // If we're a somewhat high count, and we're covering 2/3 // of the total mers, assume that there are lots of // errors (or polymorphism) that are preventing us from // covering many distinct mers. // if ((i > 25 * Xcoverage) && ((total / (double)totalUsefulAll) > (2.0 / 3.0))) maxCount = i; } fprintf(stderr, "Set maxCount to "F_U32", which will cover %.2f%% of distinct mers and %.2f%% of all mers.\n", i, 100.0 * distinct / totalUsefulDistinct, 100.0 * total / totalUsefulAll); // Pass 2: if the limit is relatively small compared to our // guessed Xcoverage, and %total is high, keep going to // close 75% of the gap in total coverage. So if the TC is // 90%, we'd keep going until TC is 97.5%. // // If we're WAY low compared to X coverage, close the gap // too, but not as much. This only happens if we're // covering 99% of the distinct, so we're already in good // shape. The genome doesn't appear to be very repetitive. // if (((maxCount < 5 * Xcoverage)) || ((maxCount < 50 * Xcoverage) && (total / (double)totalUsefulAll > 0.90))) { double closeAmount = 0.75; if (total / (double)totalUsefulAll <= 0.90) closeAmount = 0.5; // No, really. This is just 0.75 * (1-TC) + TC double desiredTC = closeAmount + (1 - closeAmount) * total / (double)totalUsefulAll; for (; (i < MF->histogramLength()) && (total / (double)totalUsefulAll < desiredTC); i++) { distinct += MF->histogram(i); total += MF->histogram(i) * i; } maxCount = i; fprintf(stderr, "Reset maxCount to "F_U32", which will cover %.2f%% of distinct mers and %.2f%% of all mers.\n", maxCount, 100.0 * distinct / totalUsefulDistinct, 100.0 * total / totalUsefulAll); } fprintf(stdout, F_U32"\n", maxCount); return(0); } canu-1.3/src/meryl/estimate-mer-threshold.mk000066400000000000000000000010121272335422300211360ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := estimate-mer-threshold SOURCES := estimate-mer-threshold.C SRC_INCDIRS := .. ../AS_UTL libleaff TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lleaff -lcanu TGT_PREREQS := libleaff.a libcanu.a SUBMAKEFILES := canu-1.3/src/meryl/gkrpt.pl000077500000000000000000000176341272335422300157270ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/AS_MER/gkrpt.pl # # Modifications by: # # Brian P. Walenz from 2008-JAN-03 to 2013-AUG-01 # are Copyright 2008,2013 J. Craig Venter Institute, and # are subject to the GNU General Public License version 2 # # Brian P. Walenz on 2015-APR-10 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my %kmers; while (!eof(STDIN)){ my $count = ; chomp $count; my $kmer = ; chomp $kmer; $count =~ s/>//g; $count = int($count); $kmer =~ tr/acgt/ACGT/; my $numbad = $kmer =~ tr/ACGT//c; if ($numbad > 0){ die "$numbad nonacgtACGT characters in kmer $kmer\n"; } if (defined($kmers{$kmer})){ die "kmer repeated in input $kmer\n"; } $kmers{$kmer} = $count; } foreach my $kmer (sort { $kmers{$b} <=> $kmers{$a} } (keys %kmers)){ my $startkmer = my $curkmer = $kmer; my $startcount = my $curcount = $kmers{$kmer}; if ($curcount < 0) { next; } else { $kmers{$kmer} = - $curcount; } my $currpt = $curkmer; for ( ; ; ) { my $nextcount = -1; my $realcount = -1; my $maxkmer; my $realkmer; my $tmpkmer; my $maxcount = -1; $curkmer = (substr $curkmer, 1) . "A"; my $rckmer = reverse $curkmer; $rckmer =~ tr/ACGT/TGCA/; if (defined($kmers{$curkmer})){ $nextcount = $kmers{$curkmer}; $tmpkmer = $curkmer; } else { if (defined($kmers{$rckmer})){ $nextcount = $kmers{$rckmer}; $tmpkmer = $rckmer; } else { $nextcount = -1; } } if ((abs $nextcount) > $maxcount){ $maxcount = abs $nextcount; $realcount = $nextcount; $realkmer = $tmpkmer; $maxkmer = $curkmer; } substr($curkmer, -1) = "C"; if (defined($kmers{$curkmer})){ $nextcount = $kmers{$curkmer}; $tmpkmer = $curkmer; } else { substr($rckmer, 0, 1) = "G"; if (defined($kmers{$rckmer})){ $nextcount = $kmers{$rckmer}; $tmpkmer = $rckmer; } else { $nextcount = -1; } } if ((abs $nextcount) > $maxcount){ $maxcount = abs $nextcount; $realcount = $nextcount; $realkmer = $tmpkmer; $maxkmer = $curkmer; } substr($curkmer, -1) = "G"; if (defined($kmers{$curkmer})){ $nextcount = $kmers{$curkmer}; $tmpkmer = $curkmer; } else { substr($rckmer, 0, 1) = "C"; if (defined($kmers{$rckmer})){ $nextcount = $kmers{$rckmer}; $tmpkmer = $rckmer; } else { $nextcount = -1; } } if ((abs $nextcount) > $maxcount){ $maxcount = abs $nextcount; $realcount = $nextcount; $realkmer = $tmpkmer; $maxkmer = $curkmer; } substr($curkmer, -1) = "T"; if (defined($kmers{$curkmer})){ $nextcount = $kmers{$curkmer}; $tmpkmer = $curkmer; } else { substr($rckmer, 0, 1) = "A"; if (defined($kmers{$rckmer})){ $nextcount = $kmers{$rckmer}; $tmpkmer = $rckmer; } else { $nextcount = -1; } } if ((abs $nextcount) > $maxcount){ $maxcount = abs $nextcount; $realcount = $nextcount; $realkmer = $tmpkmer; $maxkmer = $curkmer; } if (($realcount < 0) || ($realcount < ($curcount / 2))) { last; } else { $curkmer = $maxkmer; $curcount = $realcount; $kmers{$realkmer} = - $realcount; $currpt .= (substr $curkmer, -1); } } $curcount = $startcount; $curkmer = $startkmer; for ( ; ; ) { my $nextcount = -1; my $realcount = -1; my $maxkmer; my $realkmer; my $tmpkmer; my $maxcount = -1; $curkmer = "A" . (substr $curkmer, 0, -1); my $rckmer = reverse $curkmer; $rckmer =~ tr/ACGT/TGCA/; if (defined($kmers{$curkmer})){ $nextcount = $kmers{$curkmer}; $tmpkmer = $curkmer; } else { if (defined($kmers{$rckmer})){ $nextcount = $kmers{$rckmer}; $tmpkmer = $rckmer; } else { $nextcount = -1; } } if ((abs $nextcount) > $maxcount){ $maxcount = abs $nextcount; $realcount = $nextcount; $realkmer = $tmpkmer; $maxkmer = $curkmer; } substr($curkmer, 0, 1) = "C"; if (defined($kmers{$curkmer})){ $nextcount = $kmers{$curkmer}; $tmpkmer = $curkmer; } else { substr($rckmer, -1) = "G"; if (defined($kmers{$rckmer})){ $nextcount = $kmers{$rckmer}; $tmpkmer = $rckmer; } else { $nextcount = -1; } } if ((abs $nextcount) > $maxcount){ $maxcount = abs $nextcount; $realcount = $nextcount; $realkmer = $tmpkmer; $maxkmer = $curkmer; } substr($curkmer, 0, 1) = "G"; if (defined($kmers{$curkmer})){ $nextcount = $kmers{$curkmer}; $tmpkmer = $curkmer; } else { substr($rckmer, -1) = "C"; if (defined($kmers{$rckmer})){ $nextcount = $kmers{$rckmer}; $tmpkmer = $rckmer; } else { $nextcount = -1; } } if ((abs $nextcount) > $maxcount){ $maxcount = abs $nextcount; $realcount = $nextcount; $realkmer = $tmpkmer; $maxkmer = $curkmer; } substr($curkmer, 0, 1) = "T"; if (defined($kmers{$curkmer})){ $nextcount = $kmers{$curkmer}; $tmpkmer = $curkmer; } else { substr($rckmer, -1) = "A"; if (defined($kmers{$rckmer})){ $nextcount = $kmers{$rckmer}; $tmpkmer = $rckmer; } else { $nextcount = -1; } } if ((abs $nextcount) > $maxcount){ $maxcount = abs $nextcount; $realcount = $nextcount; $realkmer = $tmpkmer; $maxkmer = $curkmer; } if (($realcount < 0) || ($realcount < ($curcount / 2))) { last; } else { $curkmer = $maxkmer; $curcount = $realcount; $kmers{$realkmer} = - $realcount; $currpt = (substr $curkmer, 0, 1) . $currpt; } } if ((my $lenrpt = length $currpt) > $ARGV[0]) { print ">$startkmer $startcount $lenrpt\n$currpt\n"; } } canu-1.3/src/meryl/leaff-blocks.C000066400000000000000000000042521272335422300166640ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/leaff/blocks.C * * Modifications by: * * Brian P. Walenz from 2009-FEB-07 to 2014-APR-11 * are Copyright 2009,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "seqCache.H" void dumpBlocks(char *filename) { seqCache *F = 0L; seqInCore *S = 0L; bool V[256] = {0}; for (uint32 i=0; i<256; i++) V[i] = false; V['n'] = true; V['N'] = true; F = new seqCache(filename); for (uint32 s=0; sgetNumberOfSequences(); s++) { seqInCore *S = F->getSequenceInCore(s); uint32 len = S->sequenceLength(); char begseq = S->sequence()[0]; bool nnn = V[begseq]; uint32 begpos = 0; uint32 pos = 0; for (pos=0; possequence()[pos]; if (nnn != V[seq]) { fprintf(stdout, "%c "F_U32" "F_U32" "F_U32" "F_U32"\n", begseq, s, begpos, pos, pos - begpos); nnn = V[seq]; begpos = pos; begseq = seq; } } fprintf(stdout, "%c "F_U32" "F_U32" "F_U32" "F_U32"\n", begseq, s, begpos, pos, pos - begpos); fprintf(stdout, ". "F_U32" "F_U32" "F_U32"\n", s, pos, 0); delete S; } delete F; } canu-1.3/src/meryl/leaff-duplicates.C000066400000000000000000000122651272335422300175470ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/leaff/dups.C * * Modifications by: * * Brian P. Walenz from 2009-FEB-07 to 2014-APR-11 * are Copyright 2009,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "seqCache.H" #include "md5.H" md5_s * computeMD5ForEachSequence(seqCache *F) { uint32 numSeqs = F->getNumberOfSequences(); md5_s *result = new md5_s [numSeqs]; for (uint32 idx=0; idx < numSeqs; idx++) { seqInCore *s1 = F->getSequenceInCore(idx); md5_string(result+idx, s1->sequence(), s1->sequenceLength()); result[idx].i = s1->getIID(); delete s1; } return(result); } void mapDuplicates_Print(char *filea, seqInCore *sa, char *fileb, seqInCore *sb) { if (strcmp(sa->sequence(), sb->sequence()) == 0) fprintf(stdout, F_U32" <-> "F_U32"\n", sa->getIID(), sb->getIID()); else fprintf(stderr, "COLLISION DETECTED BETWEEN %s:"F_U32" AND %s:"F_U32"!\nPLEASE REPORT THIS TO bri@walenz.org!\n", filea, sa->getIID(), fileb, sb->getIID()); } void findDuplicates(char *filename) { seqInCore *s1 = 0L; seqInCore *s2 = 0L; seqCache *A = new seqCache(filename); uint32 numSeqs = A->getNumberOfSequences(); fprintf(stderr, "Computing MD5's for each sequence in '%s'.\n", filename); md5_s *result = computeMD5ForEachSequence(A); fprintf(stderr, "Sorting MD5's.\n"); qsort(result, numSeqs, sizeof(md5_s), md5_compare); fprintf(stderr, "Verifying identity, and output\n"); for (uint32 idx=1; idxgetSequenceInCore(result[idx-1].i); s2 = A->getSequenceInCore(result[idx].i); if (strcmp(s1->sequence(), s2->sequence()) == 0) { fprintf(stdout, F_U32":%s\n"F_U32":%s\n\n", result[idx-1].i, s1->header(), result[idx ].i, s2->header()); } else { fprintf(stderr, "COLLISION DETECTED BETWEEN IID "F_U32" AND "F_U32"!\nPLEASE REPORT THIS TO bri@walenz.org!\n", result[idx-1].i, result[idx].i); } delete s1; delete s2; } } delete [] result; delete A; } void mapDuplicates(char *filea, char *fileb) { fprintf(stderr, "Computing MD5's for each sequence in '%s'.\n", filea); seqCache *A = new seqCache(filea); md5_s *resultA = computeMD5ForEachSequence(A); fprintf(stderr, "Computing MD5's for each sequence in '%s'.\n", fileb); seqCache *B = new seqCache(fileb); md5_s *resultB = computeMD5ForEachSequence(B); uint32 numSeqsA = A->getNumberOfSequences(); uint32 numSeqsB = B->getNumberOfSequences(); uint32 idxA = 0; uint32 idxB = 0; fprintf(stderr, "Sorting MD5's.\n"); qsort(resultA, numSeqsA, sizeof(md5_s), md5_compare); qsort(resultB, numSeqsB, sizeof(md5_s), md5_compare); fprintf(stderr, "Finding duplicates.\n"); while ((idxAgetSequenceInCore(resultA[idxA].i); seqInCore *sb = B->getSequenceInCore(resultB[idxB].i); mapDuplicates_Print(filea, sa, fileb, sb); // While the B sequence matches the current A sequence, output a match // uint32 idxBb = idxB+1; int resb = md5_compare(resultA+idxA, resultB+idxBb); while (resb == 0) { seqInCore *sbb = B->getSequenceInCore(resultB[idxBb].i); mapDuplicates_Print(filea, sa, fileb, sbb); delete sbb; idxBb++; resb = md5_compare(resultA+idxA, resultB+idxBb); } // And likewise for A // uint32 idxAa = idxA+1; int resa = md5_compare(resultA+idxAa, resultB+idxB); while (resa == 0) { seqInCore *saa = A->getSequenceInCore(resultA[idxAa].i); mapDuplicates_Print(filea, saa, fileb, sb); delete saa; idxAa++; resa = md5_compare(resultA+idxAa, resultB+idxB); } delete sa; delete sb; idxA++; idxB++; } else { if (res < 0) idxA++; else idxB++; } } delete A; delete B; } canu-1.3/src/meryl/leaff-gc.C000066400000000000000000000110331272335422300157730ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/leaff/gc.C * * Modifications by: * * Brian P. Walenz from 2009-FEB-07 to 2014-APR-11 * are Copyright 2009,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "seqCache.H" void computeGCcontent(char *filename) { seqCache *A = new seqCache(filename); for (uint32 idx=0; idx < A->getNumberOfSequences(); idx++) { seqInCore *S = A->getSequenceInCore(idx); char *s = S->sequence(); uint32 genomeLength = S->sequenceLength(); fprintf(stdout, ">%s\n", S->header()); int gc[256] = {0}; gc['c'] = 1; gc['C'] = 1; gc['g'] = 1; gc['G'] = 1; // Replace the sequence with "g or c". We can't do this inline, // since output reports the sequence too. The extra 1000 at the // end is important, since we do not bother checking for the end // of the valid data, just assume that it's zero. // char *g = new char [S->sequenceLength() + 1000]; for (uint32 i=0; i 1) ? g[i-2] : 0); ave5 += g[i+2] - ((i > 2) ? g[i-3] : 0); ave11 += g[i+5] - ((i > 5) ? g[i-6] : 0); ave51 += g[i+25] - ((i > 25) ? g[i-25] : 0); ave101 += g[i+50] - ((i > 50) ? g[i-51] : 0); ave201 += g[i+100] - ((i > 100) ? g[i-101] : 0); ave501 += g[i+250] - ((i > 250) ? g[i-251] : 0); ave1001 += g[i+500] - ((i > 500) ? g[i-501] : 0); ave2001 += g[i+1000] - ((i > 1000) ? g[i-1001] : 0); fprintf(stdout, F_U32"\t"F_U32"\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%.2f\n", i, s[i], ave3 / (double)((i >= 1) ? 3 - ((i < genomeLength - 1) ? 0 : i + 2 - genomeLength) : i+2), ave5 / (double)((i >= 2) ? 5 - ((i < genomeLength - 2) ? 0 : i + 3 - genomeLength) : i+3), ave11 / (double)((i >= 5) ? 11 - ((i < genomeLength - 4) ? 0 : i + 5 - genomeLength) : i+6), ave51 / (double)((i >= 25) ? 51 - ((i < genomeLength - 24) ? 0 : i + 25 - genomeLength) : i+26), ave101 / (double)((i >= 50) ? 101 - ((i < genomeLength - 49) ? 0 : i + 50 - genomeLength) : i+51), ave201 / (double)((i >= 100) ? 201 - ((i < genomeLength - 99) ? 0 : i + 100 - genomeLength) : i+101), ave501 / (double)((i >= 250) ? 501 - ((i < genomeLength - 249) ? 0 : i + 250 - genomeLength) : i+251), ave1001 / (double)((i >= 500) ? 1001 - ((i < genomeLength - 499) ? 0 : i + 500 - genomeLength) : i+501), ave2001 / (double)((i >= 1000) ? 2001 - ((i < genomeLength - 999) ? 0 : i + 1000 - genomeLength) : i+1001)); } delete [] g; delete S; } } canu-1.3/src/meryl/leaff-partition.C000066400000000000000000000137421272335422300174240ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/leaff/partition.C * * Modifications by: * * Brian P. Walenz from 2009-FEB-07 to 2014-APR-11 * are Copyright 2009-2010,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "seqCache.H" #include struct partition_s { uint32 length; uint32 index; uint32 partition; }; static int partition_s_compare(const void *A, const void *B) { const partition_s *a = (const partition_s *)A; const partition_s *b = (const partition_s *)B; if (a->length < b->length) return(1); if (a->length > b->length) return(-1); return(0); } static partition_s * loadPartition(seqCache *F) { uint32 n = F->getNumberOfSequences(); partition_s *p = new partition_s [n]; for (uint32 i=0; igetSequenceLength(i); p[i].index = i; p[i].partition = 0; } qsort(p, n, sizeof(partition_s), partition_s_compare); return(p); } static void outputPartition(seqCache *F, char *prefix, partition_s *p, uint32 openP, uint32 n) { char filename[1024]; // Check that everything has been partitioned // for (uint32 i=0; igetSequenceInCore(p[i].index); fprintf(file, ">%s\n", S->header()); fwrite(S->sequence(), sizeof(char), S->sequenceLength(), file); fprintf(file, "\n"); if (S->sequenceLength() != p[i].length) { fprintf(stderr, "Huh? '%s' "F_U32" != "F_U32"\n", S->header(), S->sequenceLength(), p[i].length); } delete S; } fclose(file); } } else { // This dumps the partition information to stdout. // fprintf(stdout, F_U32"\n", openP); for (uint32 o=1; o<=openP; o++) { uint32 sizeP = 0; for (uint32 i=0; igetNumberOfSequences(); partition_s *p = loadPartition(F); uint32 openP = 1; // Currently open partition uint32 sizeP = 0; // Size of open partition uint32 seqsP = n; // Number of sequences to partition // For any sequences larger than partitionSize, create // partitions containing just one sequence // for (uint32 i=0; i partitionSize) { p[i].partition = openP++; seqsP--; } } // For the remaining, iterate through the list, // greedily placing the longest sequence that fits // into the open partition // while (seqsP > 0) { for (uint32 i=0; igetNumberOfSequences(); partition_s *p = loadPartition(F); if (partitionSize > n) partitionSize = n; // The size, in bases, of each partition // uint32 *s = new uint32 [partitionSize]; for (uint32 i=0; igetNumberOfSequences(); partition_s *p = new partition_s [n]; uint32 numSeqPerPart = (uint32)ceil(n / (double)numSegments); for (uint32 i=0; igetSequenceLength(i); p[i].index = i; p[i].partition = i / numSeqPerPart + 1; } outputPartition(F, prefix, p, numSegments, n); delete [] p; delete F; } canu-1.3/src/meryl/leaff-simulate.C000066400000000000000000000146511272335422300172360ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/leaff/simseq.C * * Modifications by: * * Brian P. Walenz from 2004-JUN-24 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2008-SEP-22 to 2009-JUN-13 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-12 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "mt19937ar.H" // This is Liliana Florea's sequencing error simulator. Bri hacked // it to use a real RNG, and to make it work from leaff. typedef struct edit_script { uint32 optype; uint32 num; struct edit_script *next; } EditScript_t; typedef struct align { uint32 offset, len; EditScript_t *script; } Align_t; // This guy is provided by leaff extern mtRandom MT; // RAND returns x numbers, starting at number y. // #define RAND(x,y) (int)((y) + (MT.mtRandom32() % (x))) #define max(x,y) ((x)>=(y) ? (x):(y)) #define min(x,y) ((x)<=(y) ? (x):(y)) #define MOV 3 #define SUB 2 #define INS 1 #define DEL 0 EditScript_t * new_script(int optype, uint32 num, EditScript_t *next) { EditScript_t *newtp = new EditScript_t; newtp->optype = optype; newtp->num = num; newtp->next = next; return(newtp); } /* DEL(pos), SUB(pos) - modifY position pos; INS - insert right before pos */ void insert(Align_t *aln, uint32 in_pos, uint32 in_optype) { uint32 i, num, optype; EditScript_t *t, *tp; //fprintf(stderr, "Modify script op=%d pos=%d\n", in_optype, in_pos); for (t=aln->script, i=0, tp=NULL; t; tp=t, t=t->next) { num = t->num; optype = t->optype; switch (optype) { case INS: if (in_pos==i+1) { if (tp) tp->next = new_script(in_optype, 1, tp->next); else aln->script = new_script(in_optype, 1, aln->script); return; } break; case DEL: i += num; break; case SUB: case MOV: if (inum = l; tp = t; tp->next = new_script(in_optype, 1, tp->next); tp = tp->next; tp->next = new_script(optype, r, tp->next); } else if (!l) { if (tp) tp->next = new_script(in_optype, 1, t); else aln->script = new_script(in_optype, 1, aln->script); if (in_optype!=INS) t->num -= 1; } else { tp = t; tp->next = new_script(in_optype, 1, tp->next); if (in_optype!=INS) t->num -= 1; } return; } i += num; break; default: fprintf(stderr, "Unrecognized optype (%d).\n", in_optype); break; } } //fprintf(stderr, "Failed to modify sequence (%d,%d).\n", in_optype, in_pos); } void print_simseq(char *seq, char *hdr, Align_t *aln, double P, uint32 CUT, uint32 COPY) { uint32 k, e; char *s; char let_4[4] = {'A','C','G','T'}; char let_3A[3] = {'C','G','T'}; char let_3C[3] = {'A','G','T'}; char let_3G[3] = {'A','C','T'}; char let_3T[3] = {'A','C','G'}; EditScript_t *t; fprintf(stdout, ">"); while ((*hdr) && !isspace(*hdr)) fprintf(stdout, "%c", *hdr++); fprintf(stdout, ":seq=%d:copy=%d:loc=%d-%d:err=%1.2f\n", CUT+1, COPY+1, aln->offset, aln->offset+aln->len-1, P); s = seq + aln->offset-1; for (t=aln->script; t; t=t->next) { if (*s == 0) break; switch (t->optype) { case INS: for (k=0; knum; k++) { e = RAND(4,0); fprintf(stdout, "%c", let_4[e]); } break; case DEL: while (*s && t->num) { s++; t->num--; } break; case SUB: for (k=0; knum; k++) { e = RAND(3,0); if (*s=='A') fprintf(stdout, "%c", let_3A[e]); else if (*s=='C') fprintf(stdout, "%c", let_3C[e]); else if (*s=='G') fprintf(stdout, "%c", let_3G[e]); else if (*s=='T') fprintf(stdout, "%c", let_3T[e]); else fprintf(stdout, "%c", 'A'); s++; } break; case MOV: for (k=0; knum; k++) { if (*s == 0) { k = t->num; } else { fprintf(stdout, "%c", *s); s++; } } break; default: fprintf(stderr, "Unrecognized optype (%d).\n", t->optype); break; } } fprintf(stdout, "\n"); } void simseq(char *seq, char *hdr, uint32 len, uint32 N, uint32 L, uint32 C, double P) { Align_t align; uint32 i, j, k; uint32 start; EditScript_t *s; for (i=0; inext; delete s; } } } } canu-1.3/src/meryl/leaff-statistics.C000066400000000000000000000100251272335422300175740ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/leaff/stats.C * * Modifications by: * * Brian P. Walenz from 2009-FEB-07 to 2014-APR-11 * are Copyright 2009,2012-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-08 to 2015-MAR-21 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "seqCache.H" #include using namespace std; void stats(char *filename, uint64 refLen) { seqCache *F = new seqCache(filename); bool V[256]; for (uint32 i=0; i<256; i++) V[i] = false; V['n'] = true; V['N'] = true; uint32 numSeq = F->getNumberOfSequences(); uint64 Ss = 0; // actual length of span uint64 Rs = 0; // reference length of span uint32 *Ls = new uint32 [numSeq]; uint64 Sb = 0; uint64 Rb = 0; uint32 *Lb = new uint32 [numSeq]; for (uint32 i=0; igetSequenceInCore(s); uint32 len = S->sequenceLength(); uint32 span = len; uint32 base = len; for (uint32 pos=1; possequence()[pos]]) base--; } Ss += span; Sb += base; Ls[S->getIID()] = span; Lb[S->getIID()] = base; delete S; } if (refLen > 0) { Rs = refLen; Rb = refLen; } else { Rs = Ss; Rb = Sb; } //qsort(Ls, numSeq, sizeof(uint32), uint32_compare); //qsort(Lb, numSeq, sizeof(uint32), uint32_compare); sort(Ls, Ls + numSeq); sort(Lb, Lb + numSeq); reverse(Ls, Ls + numSeq); reverse(Lb, Lb + numSeq); uint32 n50s[11] = {0}; uint32 l50s[11] = {0}; uint32 n50b[11] = {0}; uint32 l50b[11] = {0}; uint32 sizes[11] = {0}; uint32 sizeb[11] = {0}; for (uint32 i=0; i<11; i++) { sizes[i] = i * Rs / 10; sizeb[i] = i * Rb / 10; //fprintf(stderr, "SIZE %2d s=%d b=%d\n", i, sizes[i], sizeb[i]); } for (uint32 i=0, sum=0, n=1; (i < numSeq) && (n < 11); i++) { if ((sum < sizes[n]) && (sizes[n] <= sum + Ls[i])) { n50s[n] = Ls[i]; l50s[n] = i; n++; } sum += Ls[i]; } for (uint32 i=0, sum=0, n=1; (i < numSeq) && (n < 11); i++) { if ((sum < sizeb[n]) && (sizeb[n] <= sum + Lb[i])) { n50b[n] = Ls[i]; l50b[n] = i; n++; } sum += Lb[i]; } //for (uint32 i=0, sum=0; sum < Rb/2; i++) { //} fprintf(stdout, "%s\n", F->getSourceName()); fprintf(stdout, "\n"); fprintf(stdout, "numSeqs "F_U32"\n", numSeq); fprintf(stdout, "\n"); fprintf(stdout, "SPAN (smallest "F_U32" largest "F_U32")\n", Ls[numSeq-1], Ls[0]); for (uint32 i=1; i<10; i++) fprintf(stdout, "n"F_U32" %10"F_U32P" at index "F_U32"\n", 10 * i, n50s[i], l50s[i]); fprintf(stdout, "totLen %10"F_U64P"\n", Ss); fprintf(stdout, "refLen %10"F_U64P"\n", Rs); fprintf(stdout, "\n"); fprintf(stdout, "BASES (smallest "F_U32" largest "F_U32")\n", Lb[numSeq-1], Lb[0]); for (uint32 i=1; i<10; i++) fprintf(stdout, "n"F_U32" %10"F_U32P" at index "F_U32"\n", 10 * i, n50b[i], l50b[i]); fprintf(stdout, "totLen %10"F_U64P"\n", Sb); fprintf(stdout, "refLen %10"F_U64P"\n", Rb); delete [] Ls; delete [] Lb; } canu-1.3/src/meryl/leaff.C000066400000000000000000000706321272335422300154160ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/leaff/leaff.C * * Modifications by: * * Brian P. Walenz from 2003-JAN-02 to 2003-OCT-14 * are Copyright 2003 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-FEB-20 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAR-06 to 2014-APR-11 * are Copyright 2005-2009,2011-2012,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Liliana Florea on 2011-NOV-16 * are Copyright 2011 Liliana Florea, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-AUG-22 to 2015-JAN-13 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "seqCache.H" #include "seqStore.H" #include "md5.H" #include "mt19937ar.H" #include "dnaAlphabets.H" // Analysis functions // void dumpBlocks(char *filename); void stats(char *filename, uint64 refLen); void partitionBySize(char *prefix, uint64 partitionSize, char *filename); void partitionByBucket(char *prefix, uint64 partitionSize, char *filename); void partitionBySegment(char *prefix, uint64 numSegments, char *filename); void simseq(char *,char *, uint32, uint32, uint32, uint32, double); void computeGCcontent(char *name); void findDuplicates(char *filename); void mapDuplicates(char *filea, char *fileb); void processFile(char *filename); void processArray(int argc, char **argv); bool doReverse = false; bool doComplement = false; bool withDefLine = true; char *specialDefLine = 0L; uint32 withLineBreaks = 0; bool toUppercase = false; char translate[256] = {0}; seqCache *fasta = 0L; uint32 begPos = (uint32)0; uint32 endPos = ~(uint32)0; uint32 endExtract = ~(uint32)0; mtRandom MT; static void failIfNoSource(void) { if (fasta == 0L) fprintf(stderr, "No source file specified.\n"), exit(1); } static void failIfNotRandomAccess(void) { if (fasta->randomAccessSupported() == false) fprintf(stderr, "Algorithm required random access; soruce file not supported.\n"), exit(1); } static void helpStandard(char *program) { fprintf(stderr, "usage: %s [-f fasta-file] [options]\n", program); fprintf(stderr, "\n"); fprintf(stderr, "SOURCE FILES\n"); fprintf(stderr, " -f file: use sequence in 'file' (-F is also allowed for historical reasons)\n"); fprintf(stderr, " -A file: read actions from 'file'\n"); fprintf(stderr, "\n"); fprintf(stderr, "SOURCE FILE EXAMINATION\n"); fprintf(stderr, " -d: print the number of sequences in the fasta\n"); fprintf(stderr, " -i name: print an index, labelling the source 'name'\n"); fprintf(stderr, "\n"); fprintf(stderr, "OUTPUT OPTIONS\n"); fprintf(stderr, " -6 <#>: insert a newline every 60 letters\n"); fprintf(stderr, " (if the next arg is a number, newlines are inserted every\n"); fprintf(stderr, " n letters, e.g., -6 80. Disable line breaks with -6 0,\n"); fprintf(stderr, " or just don't use -6!)\n"); fprintf(stderr, " -e beg end: Print only the bases from position 'beg' to position 'end'\n"); fprintf(stderr, " (space based, relative to the FORWARD sequence!) If\n"); fprintf(stderr, " beg == end, then the entire sequence is printed. It is an\n"); fprintf(stderr, " error to specify beg > end, or beg > len, or end > len.\n"); fprintf(stderr, " -ends n Print n bases from each end of the sequence. One input\n"); fprintf(stderr, " sequence generates two output sequences, with '_5' or '_3'\n"); fprintf(stderr, " appended to the ID. If 2n >= length of the sequence, the\n"); fprintf(stderr, " sequence itself is printed, no ends are extracted (they\n"); fprintf(stderr, " overlap).\n"); fprintf(stderr, " -C: complement the sequences\n"); fprintf(stderr, " -H: DON'T print the defline\n"); fprintf(stderr, " -h: Use the next word as the defline (\"-H -H\" will reset to the\n"); fprintf(stderr, " original defline\n"); fprintf(stderr, " -R: reverse the sequences\n"); fprintf(stderr, " -u: uppercase all bases\n"); fprintf(stderr, "\n"); fprintf(stderr, "SEQUENCE SELECTION\n"); fprintf(stderr, " -G n s l: print n randomly generated sequences, 0 < s <= length <= l\n"); fprintf(stderr, " -L s l: print all sequences such that s <= length < l\n"); fprintf(stderr, " -N l h: print all sequences such that l <= %% N composition < h\n"); fprintf(stderr, " (NOTE 0.0 <= l < h < 100.0)\n"); fprintf(stderr, " (NOTE that you cannot print sequences with 100%% N\n"); fprintf(stderr, " This is a useful bug).\n"); fprintf(stderr, " -q file: print sequences from the seqid list in 'file'\n"); fprintf(stderr, " -r num: print 'num' randomly picked sequences\n"); fprintf(stderr, " -s seqid: print the single sequence 'seqid'\n"); fprintf(stderr, " -S f l: print all the sequences from ID 'f' to 'l' (inclusive)\n"); fprintf(stderr, " -W: print all sequences (do the whole file)\n"); fprintf(stderr, "\n"); fprintf(stderr, "LONGER HELP\n"); fprintf(stderr, " -help analysis\n"); fprintf(stderr, " -help examples\n"); } static void helpAnalysis(char *program) { fprintf(stderr, "usage: %s [-f ] [options]\n", program); fprintf(stderr, "\n"); fprintf(stderr, " --findduplicates a.fasta\n"); fprintf(stderr, " Reports sequences that are present more than once. Output\n"); fprintf(stderr, " is a list of pairs of deflines, separated by a newline.\n"); fprintf(stderr, "\n"); fprintf(stderr, " --mapduplicates a.fasta b.fasta\n"); fprintf(stderr, " Builds a map of IIDs from a.fasta and b.fasta that have\n"); fprintf(stderr, " identical sequences. Format is \"IIDa <-> IIDb\"\n"); fprintf(stderr, "\n"); fprintf(stderr, " --md5 a.fasta:\n"); fprintf(stderr, " Don't print the sequence, but print the md5 checksum\n"); fprintf(stderr, " (of the entire sequence) followed by the entire defline.\n"); fprintf(stderr, "\n"); fprintf(stderr, " --partition prefix [ n[gmk]bp | n ] a.fasta\n"); fprintf(stderr, " --partitionmap [ n[gmk]bp | n ] a.fasta\n"); fprintf(stderr, " Partition the sequences into roughly equal size pieces of\n"); fprintf(stderr, " size nbp, nkbp, nmbp or ngbp; or into n roughly equal sized\n"); fprintf(stderr, " parititions. Sequences larger that the partition size are\n"); fprintf(stderr, " in a partition by themself. --partitionmap writes a\n"); fprintf(stderr, " description of the partition to stdout; --partiton creates\n"); fprintf(stderr, " a fasta file 'prefix-###.fasta' for each partition.\n"); fprintf(stderr, " Example: -F some.fasta --partition parts 130mbp\n"); fprintf(stderr, " -F some.fasta --partition parts 16\n"); fprintf(stderr, "\n"); fprintf(stderr, " --segment prefix n a.fasta\n"); fprintf(stderr, " Splits the sequences into n files, prefix-###.fasta.\n"); fprintf(stderr, " Sequences are not reordered.\n"); fprintf(stderr, "\n"); fprintf(stderr, " --gccontent a.fasta\n"); fprintf(stderr, " Reports the GC content over a sliding window of\n"); fprintf(stderr, " 3, 5, 11, 51, 101, 201, 501, 1001, 2001 bp.\n"); fprintf(stderr, "\n"); fprintf(stderr, " --testindex a.fasta\n"); fprintf(stderr, " Test the index of 'file'. If index is up-to-date, leaff\n"); fprintf(stderr, " exits successfully, else, leaff exits with code 1. If an\n"); fprintf(stderr, " index file is supplied, that one is tested, otherwise, the\n"); fprintf(stderr, " default index file name is used.\n"); fprintf(stderr, "\n"); fprintf(stderr, " --dumpblocks a.fasta\n"); fprintf(stderr, " Generates a list of the blocks of N and non-N. Output\n"); fprintf(stderr, " format is 'base seq# beg end len'. 'N 84 483 485 2' means\n"); fprintf(stderr, " that a block of 2 N's starts at space-based position 483\n"); fprintf(stderr, " in sequence ordinal 84. A '.' is the end of sequence\n"); fprintf(stderr, " marker.\n"); fprintf(stderr, "\n"); fprintf(stderr, " --errors L N C P a.fasta\n"); fprintf(stderr, " For every sequence in the input file, generate new\n"); fprintf(stderr, " sequences including simulated sequencing errors.\n"); fprintf(stderr, " L -- length of the new sequence. If zero, the length\n"); fprintf(stderr, " of the original sequence will be used.\n"); fprintf(stderr, " N -- number of subsequences to generate. If L=0, all\n"); fprintf(stderr, " subsequences will be the same, and you should use\n"); fprintf(stderr, " C instead.\n"); fprintf(stderr, " C -- number of copies to generate. Each of the N\n"); fprintf(stderr, " subsequences will have C copies, each with different\n"); fprintf(stderr, " errors.\n"); fprintf(stderr, " P -- probability of an error.\n"); fprintf(stderr, "\n"); fprintf(stderr, " HINT: to simulate ESTs from genes, use L=500, N=10, C=10\n"); fprintf(stderr, " -- make C=10 sequencer runs of N=10 EST sequences\n"); fprintf(stderr, " of length 500bp each.\n"); fprintf(stderr, " to simulate mRNA from genes, use L=0, N=10, C=10\n"); fprintf(stderr, " to simulate reads from genomes, use L=800, N=10, C=1\n"); fprintf(stderr, " -- of course, N= should be increased to give the\n"); fprintf(stderr, " appropriate depth of coverage\n"); fprintf(stderr, "\n"); fprintf(stderr, " --stats a.fasta [refLen]\n"); fprintf(stderr, " Reports size statistics; number, N50, sum, largest.\n"); fprintf(stderr, " If 'refLen' is supplied, N50 is based on this size.\n"); fprintf(stderr, "\n"); fprintf(stderr, " --seqstore out.seqStore\n"); fprintf(stderr, " Converts the input file (-f) to a seqStore file.\n"); } static void helpExamples(char *program) { fprintf(stderr, "usage: %s [-f ] [options]\n", program); fprintf(stderr, "\n"); fprintf(stderr, "Options are ORDER DEPENDENT. Sequences are printed whenever an ACTION occurs\n"); fprintf(stderr, "on the command line. SEQUENCE OPTIONS are not reset when a sequence is printed.\n"); fprintf(stderr, "\n"); fprintf(stderr, "SEQUENCES are numbered starting at ZERO, not one.\n"); fprintf(stderr, "\n"); fprintf(stderr, " Print the first 10 bases of the fourth sequence in file 'genes':\n"); fprintf(stderr, " -f genes -e 0 10 -s 3\n"); fprintf(stderr, "\n"); fprintf(stderr, " Print the first 10 bases of the fourth and fifth sequences:\n"); fprintf(stderr, " -f genes -e 0 10 -s 3 -s 4\n"); fprintf(stderr, "\n"); fprintf(stderr, " Print the fourth and fifth sequences reverse complemented, and the sixth\n"); fprintf(stderr, " sequence forward. The second set of -R -C toggle off reverse-complement:\n"); fprintf(stderr, " -f genes -R -C -s 3 -s 4 -R -C -s 5\n"); fprintf(stderr, "\n"); fprintf(stderr, " Convert file 'genes' to a seqStore 'genes.seqStore'. The seqStore\n"); fprintf(stderr, " provides better performance with the kmer tools.\n"); fprintf(stderr, " -f genes --seqstore genes.seqStore\n"); } static void printSequence(char *def, char *seq, uint32 beg, uint32 end) { if (beg >= end) return; if ((endExtract != ~uint32ZERO) && (endExtract + endExtract < end - beg)) { char d[1024]; uint32 l = strlen(seq); sprintf(d, "%s_5", def); printSequence(d, seq, 0, endExtract); sprintf(d, "%s_3", def); printSequence(d, seq, l-endExtract, l); return; } if (specialDefLine) def = specialDefLine; if (withDefLine == false) def = 0L; uint32 limit = end - beg; char *n = new char [end - beg + 1]; char *m; if ((doReverse == false) && (doComplement == false)) { m = n; seq += beg; while (limit--) *(m++) = translate[*(seq++)]; } else if ((doReverse == true) && (doComplement == false)) { m = n + limit - 1; seq += beg; while (limit--) *(m--) = translate[*(seq++)]; } else if ((doReverse == false) && (doComplement == true)) { m = n; seq += beg; while (limit--) *(m++) = alphabet.complementSymbol(translate[*(seq++)]); } else if ((doReverse == true) && (doComplement == true)) { m = n + limit - 1; seq += beg; while (limit--) *(m--) = alphabet.complementSymbol(translate[*(seq++)]); } n[end-beg] = 0; if (def) fprintf(stdout, ">%s\n", def); if (withLineBreaks) { char *t = n; char *a = new char [withLineBreaks+1]; while (*t) { uint32 i=0; while ((*t) && (i < withLineBreaks)) a[i++] = *(t++); a[i++] = '\n'; a[i] = 0; fprintf(stdout, "%s", a); } delete [] a; } else { fprintf(stdout, "%s\n", n); } delete [] n; } static void printSequence(seqInCore *sic) { printSequence(sic->header(), sic->sequence(), (begPos!=(uint32)0) ? begPos:0, (endPos!=~uint32(0)) ? endPos:sic->sequenceLength()); } static void printSequence(uint32 sid) { seqInCore *sic = fasta->getSequenceInCore(sid); if (sic == 0L) fprintf(stderr, "WARNING: Didn't find sequence with iid '"F_U32"'\n", sid); else printSequence(sic); delete sic; } static void printSequence(char *sid) { seqInCore *sic = fasta->getSequenceInCore(sid); if (sic == 0L) fprintf(stderr, "WARNING: Didn't find sequence with name/iid '%s'\n", sid); else printSequence(sic); delete sic; } static void printIDsFromFile(char *name) { uint32 idLen = 0; uint32 idMax = 63; char *id = new char [idMax+1]; readBuffer B(name); char x = B.read(); // For optimal performance, we should sort the list of ID's given // by their IID, but the user might have a good reason for wanting // them unsorted. while (B.eof() == false) { while (alphabet.isWhitespace(x) && (B.eof() == false)) x = B.read(); if (B.eof() == false) { idLen = 0; while (!alphabet.isWhitespace(x) && (B.eof() == false)) { id[idLen++] = x; x = B.read(); if (idLen >= idMax) { idMax *= 2; char *newid = new char [idMax+1]; memcpy(newid, id, sizeof(char) * idLen); delete [] id; id = newid; } } id[idLen] = 0; seqInCore *S = fasta->getSequenceInCore(id); if (S == 0L) fprintf(stderr, "WARNING: Didn't find sequence with name/iid '%s'\n", id); else printSequence(S); } } delete [] id; } void processArray(int argc, char **argv) { int arg = 1; while (arg < argc) { if ((strcmp(argv[arg], "-f") == 0) || (strcmp(argv[arg], "-F") == 0)) { delete fasta; fasta = new seqCache(argv[++arg]); } else if (strcmp(argv[arg], "-i") == 0) { failIfNoSource(); ++arg; if ((argv[arg] == 0L) || (argv[arg][0] == '-')) fprintf(stderr, "ERROR: next arg to -i should be 'name', I got '%s'\n", (argv[arg] == 0L) ? "(nullpointer)" : argv[arg]), exit(1); for (uint32 s=0; sgetNumberOfSequences(); s++) fprintf(stdout, "G\tseq\t%s:"F_U32"\t"F_U32"\t%s\n", argv[arg], s, fasta->getSequenceLength(s), ">unimplemented"); } else if (strcmp(argv[arg], "-d") == 0) { failIfNoSource(); printf(F_U32"\n", fasta->getNumberOfSequences()); } else if (strcmp(argv[arg], "-L") == 0) { uint32 small = strtouint32(argv[++arg]); uint32 large = strtouint32(argv[++arg]); failIfNoSource(); for (uint32 s=0; sgetNumberOfSequences(); s++) if ((small <= fasta->getSequenceLength(s)) && (fasta->getSequenceLength(s) < large)) printSequence(s); } else if (strcmp(argv[arg], "-N") == 0) { double small = atof(argv[++arg]); double large = atof(argv[++arg]); failIfNoSource(); for (uint32 s=0; sgetNumberOfSequences(); s++) { seqInCore *S = fasta->getSequenceInCore(s); uint32 Ns = 0; uint32 len = S->sequenceLength(); char *seq = S->sequence(); for (uint32 i=begPos; igetNumberOfSequences(); s++) printSequence(s); } else if (strcmp(argv[arg], "-G") == 0) { uint32 n = strtouint32(argv[++arg]); uint32 s = strtouint32(argv[++arg]); uint32 l = strtouint32(argv[++arg]); char bases[4] = {'A', 'C', 'G', 'T'}; char *def = new char [1024]; char *seq = new char [l + 1]; if (s == 0) s = 1; if (s > l) fprintf(stderr, "leaff: usage: -G num-seqs min-length max-length\n"), exit(1); for (uint32 i=0; igetSequenceIID(argv[++arg]); uint32 highID = fasta->getSequenceIID(argv[++arg]); if (lowID > highID) { uint32 t = lowID; lowID = highID; highID = t; } for (uint32 s=lowID; (s <= highID) && (s <= fasta->getNumberOfSequences()); s++) printSequence(s); } else if (strcmp(argv[arg], "-r") == 0) { uint32 num = strtouint32(argv[++arg]); failIfNoSource(); failIfNotRandomAccess(); // Impossible to fix, or load whole thing into memory if (num >= fasta->getNumberOfSequences()) num = fasta->getNumberOfSequences(); uint32 *seqs = new uint32 [fasta->getNumberOfSequences()]; for (uint32 i=0; igetNumberOfSequences(); i++) seqs[i] = i; for (uint32 i=0; igetNumberOfSequences(); i++) { uint32 j = MT.mtRandom32() % (fasta->getNumberOfSequences() - i) + i; uint32 t = seqs[j]; seqs[j] = seqs[i]; seqs[i] = t; } for (uint32 i=0; igetNumberOfSequences(); s++) { seqInCore *S = fasta->getSequenceInCore(s); fprintf(stdout, "%s %s\n", md5_toascii(md5_string(&md5, S->sequence(), S->sequenceLength()), sum), S->header()); delete S; } delete fasta; exit(0); } else if ((strcmp(argv[arg], "--partition") == 0) || (strcmp(argv[arg], "--partitionmap") == 0)) { char *prefix = 0L; if (strcmp(argv[arg], "--partition") == 0) prefix = argv[++arg]; // does the next arg end with gbp, mbp, kbp or bp? If so, // partition by length, else partition into buckets. // int al = strlen(argv[arg+1]); uint64 ps = strtouint64(argv[arg+1]); char a3 = (al<3) ? '0' : alphabet.toLower(argv[arg+1][al-3]); char a2 = (al<2) ? '0' : alphabet.toLower(argv[arg+1][al-2]); char a1 = (al<1) ? '0' : alphabet.toLower(argv[arg+1][al-1]); // partition! if (!isdigit(a1) || !isdigit(a2) || !isdigit(a3)) { if ((a3 == 'g') && (a2 == 'b') && (a1 == 'p')) { ps *= 1000000000; } else if ((a3 == 'm') && (a2 == 'b') && (a1 == 'p')) { ps *= 1000000; } else if ((a3 == 'k') && (a2 == 'b') && (a1 == 'p')) { ps *= 1000; } else if (isdigit(a3) && (a2 == 'b') && (a1 == 'p')) { ps *= 1; } else { fprintf(stderr, "Unknown partition size option '%s'\n", argv[arg+1]), exit(1); } if (ps == 0) fprintf(stderr, "Unknown or zero partition size '%s'\n", argv[arg+1]), exit(1); partitionBySize(prefix, ps, argv[arg+2]); } else { if (ps == 0) fprintf(stderr, "Unknown or zero partition size '%s'\n", argv[arg+1]), exit(1); partitionByBucket(prefix, ps, argv[arg+2]); } exit(0); } else if (strcmp(argv[arg], "--segment") == 0) { partitionBySegment(argv[arg+1], strtouint32(argv[arg+2]), argv[arg+3]); exit(0); } else if (strcmp(argv[arg], "--gccontent") == 0) { computeGCcontent(argv[++arg]); exit(0); } else if (strcmp(argv[arg], "--dumpblocks") == 0) { dumpBlocks(argv[++arg]); exit(0); } else if (strcmp(argv[arg], "--stats") == 0) { stats(argv[arg+1], (argv[arg+2] != 0L) ? strtouint64(argv[arg+2]) : 0); exit(0); } else if (strcmp(argv[arg], "--errors") == 0) { uint32 L = strtouint32(argv[++arg]); // Desired length uint32 l = 0; // min of desired length, length of sequence uint32 N = strtouint32(argv[++arg]); // number of copies per sequence uint32 C = strtouint32(argv[++arg]); // number of mutations per copy double P = atof(argv[++arg]); // probability of mutation uint32 i = 0; fasta = new seqCache(argv[++arg]); seqInCore *S = fasta->getSequenceInCore(i++); while (S) { char *seq = S->sequence(); char *hdr = S->header(); uint32 len = S->sequenceLength(); l = len; if ((L > 0) && (L < len)) l = L; simseq(seq, hdr, len, N, l, C, P); delete S; S = fasta->getSequenceInCore(i++); } delete fasta; exit(0); } else if (strcmp(argv[arg], "--seqstore") == 0) { constructSeqStore(argv[++arg], fasta); exit(0); } else if (strcmp(argv[arg], "-help") == 0) { if ((argv[arg+1]) && (strcmp(argv[arg+1], "analysis") == 0)) helpAnalysis(argv[0]); else if ((argv[arg+1]) && (strcmp(argv[arg+1], "examples") == 0)) helpExamples(argv[0]); else helpStandard(argv[0]); exit(0); } else { helpStandard(argv[0]); fprintf(stderr, "Unknown option '%s'\n", argv[arg]); exit(1); } arg++; } delete fasta; fasta = 0L; } void processFile(char *filename) { FILE *F = NULL; if (strcmp(filename, "-") == 0) { F = stdin; } else { errno = 0; F = fopen(filename, "r"); if (errno) fprintf(stderr, "Couldn't open '%s': %s\n", filename, strerror(errno)), exit(1); } uint64 max = 16 * 1024 * 1024; uint64 pos = 0; size_t len = 0; char *data = new char [max]; // Suck the file into 'data' while (!feof(F)) { errno = 0; len = fread(data+pos, 1, max - pos, F); if (errno) fprintf(stderr, "Couldn't read "F_U64" bytes from '%s': %s\n", (uint64)(max-pos), filename, strerror(errno)), exit(1); pos += len; if (pos >= max) { max += 16 * 1024 * 1024; char *tmpd = new char [max]; memcpy(tmpd, data, pos); delete [] data; data = tmpd; } } if (strcmp(filename, "-") != 0) fclose(F); len = pos; // (over)count the number of words; we start at two, since the // first arg is the name of the program, and if there is only one // word and no whitespace in the file, the below loop fails to // count the second word. int argc = 2; char **argv = 0L; for (uint32 i=0; i' denoting a defline the first // thing in the file. // // Use of a readBuffer here is a bit heavyweight, but it's safe and // easy. Opening a fastaFile isn't, after all, lightweight anyway. // fastaFile *f = 0L; readBuffer *r = new readBuffer(filename); char x = r->read(); while ((r->eof() == false) && (alphabet.isWhitespace(x) == true)) x = r->read(); // If we get a fasta record separator assume it's a fasta file. If // it's eof, the file is empty, and we might as well return this // fasta file and let the client deal with the lack of sequence. // if ((x == '>') || (r->eof() == true)) f = new fastaFile(filename); delete r; return(f); } uint32 fastaFile::find(const char *sequencename) { char *ptr = _names; // If this proves far too slow, rewrite the _names string to // separate IDs with 0xff, then use strstr on the whole thing. To // find the ID, scan down the string counting the number of 0xff's. // // Similar code is used for seqStore::find() for (uint32 iid=0; iid < _header._numberOfSequences; iid++) { //fprintf(stderr, "fastaFile::find()-- '%s' vs '%s'\n", sequencename, ptr); if (strcmp(sequencename, ptr) == 0) return(iid); while (*ptr) ptr++; ptr++; } return(~uint32ZERO); } uint32 fastaFile::getSequenceLength(uint32 iid) { #ifdef DEBUG fprintf(stderr, "fastaFile::getSequenceLength()-- "F_U32"\n", iid); #endif return((iid < _numberOfSequences) ? _index[iid]._seqLength : 0); } bool fastaFile::getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax) { #ifdef DEBUG fprintf(stderr, "fastaFile::getSequence(full)-- "F_U32"\n", iid); #endif // Assume there is no index. Without being horribly complicated // (as in the previous versions of this codebase) all we'd get from // having an index around is the length of the sequence. // // Previous versions used to use the index to tell if the sequence // was squeezed (and so a direct copy to the output), if it was // fixed width (mostly direct copies) or unknown. Now we just // assume it's unknown and go byte by byte. If speed is a concern, // use the seqFile instead. if (iid >= _header._numberOfSequences) { fprintf(stderr, "fastaFile::getSequence(full)-- iid "F_U32" more than number of sequences "F_U32"\n", iid, _header._numberOfSequences); return(false); } if (sMax == 0) { sMax = 2048; s = new char [sMax]; } if (hMax == 0) { hMax = 2048; h = new char [hMax]; } if ((_index) && (sMax < _index[iid]._seqLength)) { sMax = _index[iid]._seqLength; delete [] s; s = new char [sMax]; } hLen = 0; sLen = 0; #ifdef DEBUG fprintf(stderr, "fastaFile::getSequence(full)-- seek to iid="F_U32" at pos="F_U32"\n", iid, _index[iid]._seqPosition); #endif _rb->seek(_index[iid]._seqPosition); char x = _rb->read(); // Skip whitespace at the start of the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // We should be at a '>' character now. Fail if not. if (_rb->eof()) return(false); if (x != '>') fprintf(stderr, "fastaFile::getSequence(full)-- ERROR1: In %s, expected '>' at beginning of defline, got '%c' instead.\n", _filename, x), exit(1); // Skip the '>' in the defline x = _rb->read(); // Skip whitespace between the '>' and the defline while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true) && (x != '\r') && (x != '\n')) x = _rb->read(); // Copy the defline, until the first newline. while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) { h[hLen++] = x; if (hLen >= hMax) { hMax += 2048; char *H = new char [hMax]; memcpy(H, h, hLen); delete [] h; h = H; } x = _rb->read(); } h[hLen] = 0; // Skip whitespace between the defline and the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // Copy the sequence, until EOF or the next '>'. while ((_rb->eof() == false) && (_rb->peek() != '>')) { if (alphabet.isWhitespace(x) == false) { s[sLen++] = x; if (sLen >= sMax) { if (sMax == 4294967295) // 4G - 1 fprintf(stderr, "fastaFile::getSequence()-- ERROR: sequence is too long; must be less than 4 Gbp.\n"), exit(1); if (sMax >= 2147483648) // 2G sMax = 4294967295; else sMax *= 2; char *S = new char [sMax]; memcpy(S, s, sLen); delete [] s; s = S; } } x = _rb->read(); } s[sLen] = 0; _nextID++; return(true); } // slow bool fastaFile::getSequence(uint32 iid, uint32 bgn, uint32 end, char *s) { if (iid >= _header._numberOfSequences) { fprintf(stderr, "fastaFile::getSequence(part)-- iid "F_U32" more than number of sequences "F_U32"\n", iid, _header._numberOfSequences); return(false); } #ifdef DEBUG fprintf(stderr, "fastaFile::getSequence(part)-- "F_U32"\n", iid); #endif // It is impossible to be efficient here; see the big comment in // the other getSequence() above. // // We can't even guess where to start scanning the sequence; we // just don't have any information about how much whitespace is in // the sequence. _rb->seek(_index[iid]._seqPosition); uint32 pos = 0; char x = _rb->read(); // Skip whitespace at the start of the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // We should be at a '>' character now. Fail if not. if (_rb->eof()) return(false); if (x != '>') fprintf(stderr, "fastaFile::getSequence(part)-- ERROR2: In %s, expected '>' at beginning of defline, got '%c' instead.\n", _filename, x), exit(1); // Skip the defline. while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) x = _rb->read(); // Skip whitespace between the defline and the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // Skip sequence up until bgn. while ((_rb->eof() == false) && (pos < bgn)) { if (alphabet.isWhitespace(x) == false) pos++; x = _rb->read(); } // Copy sequence while ((_rb->eof() == false) && (pos < end)) { if (alphabet.isWhitespace(x) == false) s[pos++ - bgn] = x; x = _rb->read(); } s[pos - bgn] = 0; // Fail if we didn't copy enough stuff. return((pos == end) ? true : false); } void fastaFile::clear(void) { memset(_filename, 0, FILENAME_MAX); memset(_typename, 0, FILENAME_MAX); strcpy(_typename, "FastA"); _numberOfSequences = 0; _rb = 0L; memset(&_header, 0, sizeof(fastaFileHeader)); _index = 0L; _names = 0L; _nextID = 0; } void fastaFile::loadIndex(char *indexname) { struct stat fastastat; if (AS_UTL_fileExists(indexname) == false) return; errno = 0; if (stat(_filename, &fastastat)) { fprintf(stderr, "fastaFile::constructIndex()-- stat of file '%s' failed: %s\n", _filename, strerror(errno)); return; } FILE *I = fopen(indexname, "r"); if (errno) { fprintf(stderr, "fastaFile::constructIndex()-- open of file '%s' failed: %s\n", indexname, strerror(errno)); return; } fread(&_header, sizeof(fastaFileHeader), 1, I); if ((_header._magic[0] != FASTA_MAGICNUMBER1) && (_header._magic[1] != FASTA_MAGICNUMBER2)) { fprintf(stderr, "fastaFile::constructIndex()-- magic mismatch.\n"); fclose(I); return; } #if 0 (_header._fastaModificationTime != (uint64)fastastat.st_mtime) (_header._fastaCreationTime != (uint64)fastastat.st_ctime) #endif if (_header._fastaFileSize != (uint64)fastastat.st_size) { fprintf(stderr, "fastaFile::constructIndex()-- stat mismatch.\n"); fclose(I); return; } _index = new fastaFileIndex [_header._numberOfSequences]; _names = new char [_header._namesLength]; fread(_index, sizeof(fastaFileIndex), _header._numberOfSequences, I); fread(_names, sizeof(char), _header._namesLength, I); #ifdef DEBUG fprintf(stderr, "fastaFile::constructIndex()-- '%s' LOADED\n", _filename); #endif fclose(I); return; } void fastaFile::constructIndex(void) { if (_index) return; // If the filename ends in '.fasta' then append a 'idx', // otherwise, append '.fastaidx'. char indexname[FILENAME_MAX]; strcpy(indexname, _filename); uint32 l = strlen(_filename); if ((l > 5) && (strcmp(_filename + l - 6, ".fasta") == 0)) strcat(indexname, "idx"); else strcat(indexname, ".fastaidx"); // If the index exists, suck it in and return. loadIndex(indexname); if (_index) return; #ifdef DEBUG fprintf(stderr, "fastaFile::constructIndex()-- '%s' BUILDING\n", _filename); #endif // Allocate some space for the index structures. uint32 indexMax = 64 * 1024 * 1024 / sizeof(fastaFileIndex); uint32 indexLen = 0; _index = new fastaFileIndex [indexMax]; uint32 namesMax = 32 * 1024 * 1024; uint32 namesLen = 0; _names = new char [namesMax]; // Some local storage uint64 seqStart; uint32 seqLen; uint32 seqLenMax = ~uint32ZERO; uint32 namePos; readBuffer ib(_filename); char x = ib.read(); #ifdef DEBUGINDEX fprintf(stderr, "readBuffer '%s' eof=%d x=%c %d\n", _filename, ib.eof(), x, x); #endif // Build it. // Skip whitespace at the start of the sequence. while ((ib.eof() == false) && (alphabet.isWhitespace(x) == true)) { #ifdef DEBUGINDEX fprintf(stderr, "skip '%c' %d\n", x, x); #endif x = ib.read(); } while (ib.eof() == false) { #ifdef DEBUGINDEX fprintf(stderr, "index\n"); #endif // We should be at a '>' character now. Fail if not. if (x != '>') fprintf(stderr, "fastaFile::constructIndex()-- ERROR3: In %s, expected '>' at beginning of defline, got '%c' instead.\n", _filename, x), exit(1); // Save info - ib's position is correctly at the first letter in // the defline (which might be whitespace), but the reader // expects our position to be at the '>' -- hence the -1. seqStart = ib.tell() - 1; seqLen = 0; namePos = namesLen; // Read that first letter x = ib.read(); // Copy the name to the names while ((ib.eof() == false) && (alphabet.isWhitespace(x) == false)) { if (namesLen + 1 >= namesMax) { namesMax += 32 * 1024 * 1024; char *nt = new char [namesMax]; memcpy(nt, _names, namesLen); delete [] _names; _names = nt; } _names[namesLen++] = x; #ifdef DEBUGINDEX fprintf(stderr, "name += %c\n", x); #endif x = ib.read(); } if (namesLen + 1 >= namesMax) { namesMax += 32 * 1024 * 1024; char *nt = new char [namesMax]; memcpy(nt, _names, namesLen); delete [] _names; _names = nt; } _names[namesLen++] = 0; // Skip the rest of the defline while ((ib.eof() == false) && (x != '\r') && (x != '\n')) { #ifdef DEBUGINDEX fprintf(stderr, "skip let %c\n", x); #endif x = ib.read(); } // Skip whitespace between the defline and the sequence. while ((ib.eof() == false) && (alphabet.isWhitespace(x) == true)) { #ifdef DEBUGINDEX fprintf(stderr, "skip num %d\n", x); #endif x = ib.read(); } #ifdef DEBUGINDEX fprintf(stderr, "x=%c peek=%c\n", x, ib.peek()); #endif // Count sequence length while ((ib.eof() == false) && (ib.peek() != '>')) { #ifdef DEBUGINDEX fprintf(stderr, "seqlen %s %c\n", (alphabet.isWhitespace(x) == false) ? "save" : "skip", x); #endif if (alphabet.isWhitespace(x) == false) seqLen++; if (seqLen >= seqLenMax) fprintf(stderr, "fastaFile::constructIndex()-- ERROR: In %s, sequence '%s' is too long. Maximum length is %u bases.\n", _filename, _names + namePos, seqLenMax), exit(1); x = ib.read(); } // Save to the index. if (indexLen >= indexMax) { indexMax *= 2; fastaFileIndex *et = new fastaFileIndex[indexMax]; memcpy(et, _index, sizeof(fastaFileIndex) * indexLen); delete [] _index; _index = et; } _index[indexLen]._seqPosition = seqStart; _index[indexLen]._seqLength = seqLen; #ifdef DEBUG fprintf(stderr, "INDEX iid="F_U32" len="F_U32" pos="F_U64"\n", indexLen, seqLen, seqStart); #endif indexLen++; // Load the '>' for the next iteration. x = ib.read(); } // Fill out the index meta data struct stat fastastat; errno = 0; if (stat(_filename, &fastastat)) fprintf(stderr, "fastaFile::constructIndex()-- stat() of file '%s' failed: %s\n", _filename, strerror(errno)), exit(1); _header._magic[0] = FASTA_MAGICNUMBER1; _header._magic[1] = FASTA_MAGICNUMBER2; _header._numberOfSequences = indexLen; _header._namesLength = namesLen; _header._fastaFileSize = fastastat.st_size; _header._fastaModificationTime = fastastat.st_mtime; _header._fastaCreationTime = fastastat.st_ctime; // Dump the index, if possible. errno = 0; FILE *I = fopen(indexname, "w"); if (errno) return; fwrite(&_header, sizeof(fastaFileHeader), 1, I); fwrite( _index, sizeof(fastaFileIndex), _header._numberOfSequences, I); fwrite( _names, sizeof(char), _header._namesLength, I); fclose(I); } canu-1.3/src/meryl/libleaff/fastaFile.H000066400000000000000000000051531272335422300200040ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef FASTAFILE_H #define FASTAFILE_H #include "seqFile.H" struct fastaFileHeader { uint64 _magic[2]; uint32 _numberOfSequences; // Number of sequences in the file uint32 _namesLength; // Bytes in the names uint64 _fastaFileSize; // st_size - size of file in bytes uint64 _fastaModificationTime; // st_mtime - time of last data modification uint64 _fastaCreationTime; // st_ctime - time of last file status change }; struct fastaFileIndex { uint64 _seqPosition; // Position of the sequence in the file uint32 _seqLength; // Length of the sequence (no whitespace counted) }; class fastaFile : public seqFile { protected: fastaFile(const char *filename); fastaFile(); public: ~fastaFile(); protected: seqFile *openFile(const char *filename); public: uint32 find(const char *sequencename); uint32 getSequenceLength(uint32 iid); bool getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax); bool getSequence(uint32 iid, uint32 bgn, uint32 end, char *s); private: void clear(void); void loadIndex(char *indexname); void constructIndex(void); readBuffer *_rb; fastaFileHeader _header; fastaFileIndex *_index; char *_names; uint32 _nextID; // Next sequence in the read buffer uint32 _gs_iid; uint32 _gs_pos; friend class seqFactory; }; #endif // FASTAFILE_H canu-1.3/src/meryl/libleaff/fastaStdin.C000066400000000000000000000153231272335422300202010ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "fastaStdin.H" #include "dnaAlphabets.H" fastaStdin::fastaStdin(const char *filename) { clear(); #ifdef DEBUG fprintf(stderr, "fastaStdin::fastaStdin()-- '%s'\n", (filename) ? filename : "NULLPOINTER"); #endif if (filename == 0L) { strcpy(_filename, "(stdin)"); _rb = new readBuffer("-"); } else { _pipe = popen(filename, "r"); _rb = new readBuffer(_pipe); } } fastaStdin::fastaStdin() { clear(); } fastaStdin::~fastaStdin() { delete _rb; delete [] _header; delete [] _sequence; } seqFile * fastaStdin::openFile(const char *filename) { #ifdef DEBUG fprintf(stderr, "fastaStdin::openFile()-- '%s'\n", (filename) ? filename : "NULLPOINTER"); #endif if (((filename == 0L) && (isatty(fileno(stdin)) == 0)) || ((filename != 0L) && (filename[0] == '-') && (filename[1] == 0))) return(new fastaStdin(0L)); if (filename == 0L) return(0L); // The stdin variants also handle compressed inputs (because we can't seek in these). uint32 fl = strlen(filename); char cmd[32 + fl]; if ((filename[fl-3] == '.') && (filename[fl-2] == 'g') && (filename[fl-1] == 'z')) sprintf(cmd, "gzip -dc %s", filename); else if ((filename[fl-4] == '.') && (filename[fl-3] == 'b') && (filename[fl-2] == 'z') && (filename[fl-1] == '2')) sprintf(cmd, "bzip2 -dc %s", filename); else if ((filename[fl-3] == '.') && (filename[fl-2] == 'x') && (filename[fl-1] == 'z')) sprintf(cmd, "xz -dc %s", filename); else return(0L); return(new fastaStdin(cmd)); } uint32 fastaStdin::getNumberOfSequences(void) { if (_rb->peek() == 0) return(_nextIID); else return(_nextIID + 1); } uint32 fastaStdin::find(const char *sequencename) { fprintf(stderr, "fastaStdin::find()-- ERROR! Used for random access on sequence '%s'.\n", sequencename); assert(0); return(~uint32ZERO); } uint32 fastaStdin::getSequenceLength(uint32 iid) { if (iid == _nextIID) if (loadNextSequence(_header, _headerLen, _headerMax, _sequence, _sequenceLen, _sequenceMax) == false) return(0); if (iid + 1 != _nextIID) { fprintf(stderr, "fastaStdin::getSequenceLength()-- ERROR! Used for random access. Requested iid=%u, at iid=%u\n", iid, _nextIID); assert(0); } return(strlen(_sequence)); } bool fastaStdin::getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax) { bool ret = true; #ifdef DEBUG fprintf(stderr, "fastaStdin::getSequence(full)-- "F_U32"\n", iid); #endif if (iid == _nextIID) if (loadNextSequence(_header, _headerLen, _headerMax, _sequence, _sequenceLen, _sequenceMax) == false) return(false); if (iid + 1 != _nextIID) { fprintf(stderr, "fastaStdin::getSequence(full)-- ERROR! Used for random access. Requested iid=%u, at iid=%u\n", iid, _nextIID); assert(0); } if (hLen < _headerMax) { delete [] h; hMax = _headerMax; h = new char [hMax]; } if (sLen < _sequenceMax) { delete [] s; sMax = _sequenceMax; s = new char [sMax]; } memcpy(h, _header, _headerLen + 1); hLen = _headerLen; memcpy(s, _sequence, _sequenceLen + 1); sLen = _sequenceLen; return(true); } bool fastaStdin::getSequence(uint32 iid, uint32 bgn, uint32 end, char *UNUSED(s)) { fprintf(stderr, "fastaStdin::getSequence(part)-- ERROR! Used for random access on sequence %u bgn %u end %u.\n", iid, bgn, end); assert(0); return(false); } void fastaStdin::clear(void) { memset(_filename, 0, FILENAME_MAX); memset(_typename, 0, FILENAME_MAX); _randomAccessSupported = false; strcpy(_typename, "FastAstream"); _numberOfSequences = ~uint32ZERO; _rb = 0L; _nextIID = 0; _pipe = 0L; _header = 0L; _headerLen = 0; _headerMax = 0; _sequence = 0L; _sequenceLen = 0; _sequenceMax = 0; } bool fastaStdin::loadNextSequence(char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax) { if (hMax == 0) { hMax = 2048; h = new char [hMax]; } if (sMax == 0) { sMax = 2048; s = new char [sMax]; } hLen = 0; sLen = 0; char x = _rb->read(); // Skip whitespace at the start of the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // We should be at a '>' character now. Fail if not. if (_rb->eof() == true) return(false); if (x != '>') fprintf(stderr, "fastaStdin::loadNextSequence(part)-- ERROR: In %s, expected '>' at beginning of defline, got '%c' instead.\n", _filename, x), exit(1); // Skip the '>' in the defline x = _rb->read(); // Skip whitespace between the '>' and the defline while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true) && (x != '\r') && (x != '\n')) x = _rb->read(); // Copy the defline, until the first newline. while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) { h[hLen++] = x; if (hLen >= hMax) { //fprintf(stderr, "realloc header\n"); hMax += 2048; char *H = new char [hMax]; memcpy(H, h, hLen); delete [] h; h = H; } x = _rb->read(); } h[hLen] = 0; // Skip whitespace between the defline and the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // Copy the sequence, until EOF or the next '>'. while ((_rb->eof() == false) && (_rb->peek() != '>')) { if (alphabet.isWhitespace(x) == false) { s[sLen++] = x; if (sLen >= sMax) { //fprintf(stderr, "realloc sequence\n"); sMax *= 2; char *S = new char [sMax]; memcpy(S, s, sLen); delete [] s; s = S; } } x = _rb->read(); } s[sLen] = 0; _nextIID++; return(true); } canu-1.3/src/meryl/libleaff/fastaStdin.H000066400000000000000000000042211272335422300202010ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef FASTASTDIN_H #define FASTASTDIN_H #include "seqFile.H" class fastaStdin : public seqFile { protected: fastaStdin(const char *filename); fastaStdin(); public: ~fastaStdin(); protected: seqFile *openFile(const char *filename); public: uint32 getNumberOfSequences(void); public: uint32 find(const char *sequencename); uint32 getSequenceLength(uint32 iid); bool getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax); bool getSequence(uint32 iid, uint32 bgn, uint32 end, char *s); private: void clear(void); bool loadNextSequence(char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax); readBuffer *_rb; uint32 _nextIID; FILE *_pipe; char *_header; uint32 _headerLen; uint32 _headerMax; char *_sequence; uint32 _sequenceLen; uint32 _sequenceMax; friend class seqFactory; }; #endif // FASTASTDIN_H canu-1.3/src/meryl/libleaff/fastqFile.C000066400000000000000000000375371272335422300200320ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "fastqFile.H" #include "dnaAlphabets.H" // Says 'kmerFastaFileIdx' #define FASTQ_MAGICNUMBER1 0x7473614672656d6bULL #define FASTQ_MAGICNUMBER2 0x786449656c694661ULL fastqFile::fastqFile(const char *filename) { clear(); #ifdef DEBUG fprintf(stderr, "fastqFile::fastqFile()-- '%s'\n", (filename) ? filename : "NULLPOINTER"); #endif strcpy(_filename, filename); constructIndex(); _rb = new readBuffer(_filename); _numberOfSequences = _header._numberOfSequences; } fastqFile::fastqFile() { clear(); } fastqFile::~fastqFile() { delete _rb; delete [] _index; delete [] _names; } seqFile * fastqFile::openFile(const char *filename) { struct stat st; #ifdef DEBUG fprintf(stderr, "fastqFile::openFile()-- '%s'\n", (filename) ? filename : "NULLPOINTER"); #endif if (((filename == 0L) && (isatty(fileno(stdin)) == 0)) || ((filename != 0L) && (filename[0] == '-') && (filename[1] == 0))) return(0L); errno = 0; stat(filename, &st); if (errno) return(0L); if ((st.st_mode & S_IFREG) == 0) return(0L); // Otherwise, open and see if we can get the first sequence. We // assume it's fastq if we find a '>' denoting a defline the first // thing in the file. // // Use of a readBuffer here is a bit heavyweight, but it's safe and // easy. Opening a fastqFile isn't, after all, lightweight anyway. // fastqFile *f = 0L; readBuffer *r = new readBuffer(filename); char x = r->read(); while ((r->eof() == false) && (alphabet.isWhitespace(x) == true)) x = r->read(); // If we get a fastq record separator assume it's a fastq file. If // it's eof, the file is empty, and we might as well return this // fastq file and let the client deal with the lack of sequence. // if ((x == '@') || (r->eof() == true)) f = new fastqFile(filename); delete r; return(f); } uint32 fastqFile::find(const char *sequencename) { char *ptr = _names; // If this proves far too slow, rewrite the _names string to // separate IDs with 0xff, then use strstr on the whole thing. To // find the ID, scan down the string counting the number of 0xff's. // // Similar code is used for seqStore::find() for (uint32 iid=0; iid < _header._numberOfSequences; iid++) { //fprintf(stderr, "fastqFile::find()-- '%s' vs '%s'\n", sequencename, ptr); if (strcmp(sequencename, ptr) == 0) return(iid); while (*ptr) ptr++; ptr++; } return(~uint32ZERO); } uint32 fastqFile::getSequenceLength(uint32 iid) { #ifdef DEBUG fprintf(stderr, "fastqFile::getSequenceLength()-- "F_U32"\n", iid); #endif return((iid < _numberOfSequences) ? _index[iid]._seqLength : 0); } bool fastqFile::getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax) { #ifdef DEBUG fprintf(stderr, "fastqFile::getSequence(full)-- "F_U32"\n", iid); #endif if (iid >= _header._numberOfSequences) { fprintf(stderr, "fastqFile::getSequence(full)-- iid "F_U32" more than number of sequences "F_U32"\n", iid, _header._numberOfSequences); return(false); } if (sMax == 0) { sMax = 2048; s = new char [sMax]; } if (hMax == 0) { hMax = 2048; h = new char [hMax]; } if ((_index) && (sMax < _index[iid]._seqLength)) { sMax = _index[iid]._seqLength; delete [] s; s = new char [sMax]; } hLen = 0; sLen = 0; #ifdef DEBUG fprintf(stderr, "fastqFile::getSequence(full)-- seek to iid="F_U32" at pos="F_U32"\n", iid, _index[iid]._seqPosition); #endif _rb->seek(_index[iid]._seqPosition); char x = _rb->read(); // Skip whitespace at the start of the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // We should be at a '@' character now. Fail if not. if (_rb->eof()) return(false); if (x != '@') fprintf(stderr, "fastqFile::getSequence(full)-- ERROR1: In %s, expected '@' at beginning of defline, got '%c' instead.\n", _filename, x), exit(1); // Skip the '@' in the defline x = _rb->read(); // Skip whitespace between the '@' and the defline while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true) && (x != '\r') && (x != '\n')) x = _rb->read(); // Copy the defline, until the first newline. while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) { h[hLen++] = x; if (hLen >= hMax) { hMax += 2048; char *H = new char [hMax]; memcpy(H, h, hLen); delete [] h; h = H; } x = _rb->read(); } h[hLen] = 0; // Skip whitespace between the defline and the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // Copy the sequence, until EOF or the start of the QV bases. while ((_rb->eof() == false) && (x != '+')) { if (alphabet.isWhitespace(x) == false) { s[sLen++] = x; if (sLen >= sMax) { if (sMax == 4294967295) // 4G - 1 fprintf(stderr, "fastqFile::getSequence()-- ERROR: sequence is too long; must be less than 4 Gbp.\n"), exit(1); if (sMax >= 2147483648) // 2G sMax = 4294967295; else sMax *= 2; char *S = new char [sMax]; memcpy(S, s, sLen); delete [] s; s = S; } } x = _rb->read(); } s[sLen] = 0; // Skip the rest of the QV id line and then the entire QV line. //x = _rb->read(); assert((_rb->eof() == true) || (x == '+')); while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) x = _rb->read(); x = _rb->read(); while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) x = _rb->read(); _nextID++; return(true); } // slow bool fastqFile::getSequence(uint32 iid, uint32 bgn, uint32 end, char *s) { if (iid >= _header._numberOfSequences) { fprintf(stderr, "fastqFile::getSequence(part)-- iid "F_U32" more than number of sequences "F_U32"\n", iid, _header._numberOfSequences); return(false); } #ifdef DEBUG fprintf(stderr, "fastqFile::getSequence(part)-- "F_U32"\n", iid); #endif // Unlike the fasta version of this, we know that all the sequence is on one line. However, we // expect fastq sequences to be small, and we still do the same processing -- character by character. _rb->seek(_index[iid]._seqPosition); uint32 pos = 0; char x = _rb->read(); // Skip whitespace at the start of the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // We should be at a '@' character now. Fail if not. if (_rb->eof()) return(false); if (x != '@') fprintf(stderr, "fastqFile::getSequence(part)-- ERROR2: In %s, expected '@' at beginning of defline, got '%c' instead.\n", _filename, x), exit(1); // Skip the defline. while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) x = _rb->read(); // Skip whitespace between the defline and the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // Skip sequence up until bgn. while ((_rb->eof() == false) && (pos < bgn)) { if (alphabet.isWhitespace(x) == false) pos++; x = _rb->read(); } // Copy sequence while ((_rb->eof() == false) && (pos < end)) { if (alphabet.isWhitespace(x) == false) s[pos++ - bgn] = x; x = _rb->read(); } s[pos - bgn] = 0; // Fail if we didn't copy enough stuff. return((pos == end) ? true : false); } void fastqFile::clear(void) { memset(_filename, 0, FILENAME_MAX); memset(_typename, 0, FILENAME_MAX); strcpy(_typename, "Fastq"); _numberOfSequences = 0; _rb = 0L; memset(&_header, 0, sizeof(fastqFileHeader)); _index = 0L; _names = 0L; _nextID = 0; } void fastqFile::loadIndex(char *indexname) { struct stat fastqstat; if (AS_UTL_fileExists(indexname) == false) return; errno = 0; if (stat(_filename, &fastqstat)) { fprintf(stderr, "fastqFile::constructIndex()-- stat of file '%s' failed: %s\n", _filename, strerror(errno)); return; } FILE *I = fopen(indexname, "r"); if (errno) { fprintf(stderr, "fastqFile::constructIndex()-- open of file '%s' failed: %s\n", indexname, strerror(errno)); return; } fread(&_header, sizeof(fastqFileHeader), 1, I); if ((_header._magic[0] != FASTQ_MAGICNUMBER1) && (_header._magic[1] != FASTQ_MAGICNUMBER2)) { fprintf(stderr, "fastqFile::constructIndex()-- magic mismatch.\n"); fclose(I); return; } if ((_header._fastqFileSize != (uint64)fastqstat.st_size) || (_header._fastqModificationTime != (uint64)fastqstat.st_mtime) || (_header._fastqCreationTime != (uint64)fastqstat.st_ctime)) { fprintf(stderr, "fastqFile::constructIndex()-- stat mismatch.\n"); fclose(I); return; } _index = new fastqFileIndex [_header._numberOfSequences]; _names = new char [_header._namesLength]; fread(_index, sizeof(fastqFileIndex), _header._numberOfSequences, I); fread(_names, sizeof(char), _header._namesLength, I); #ifdef DEBUG fprintf(stderr, "fastqFile::constructIndex()-- '%s' LOADED\n", _filename); #endif fclose(I); return; } void fastqFile::constructIndex(void) { if (_index) return; // If the filename ends in '.fastq' then append a 'idx', // otherwise, append '.fastqidx'. char indexname[FILENAME_MAX]; strcpy(indexname, _filename); uint32 l = strlen(_filename); if ((l > 5) && (strcmp(_filename + l - 6, ".fastq") == 0)) strcat(indexname, "idx"); else strcat(indexname, ".fastqidx"); // If the index exists, suck it in and return. loadIndex(indexname); if (_index) return; #ifdef DEBUG fprintf(stderr, "fastqFile::constructIndex()-- '%s' BUILDING\n", _filename); #endif // Allocate some space for the index structures. uint32 indexMax = 64 * 1024 * 1024 / sizeof(fastqFileIndex); uint32 indexLen = 0; _index = new fastqFileIndex [indexMax]; uint32 namesMax = 32 * 1024 * 1024; uint32 namesLen = 0; _names = new char [namesMax]; // Some local storage uint64 seqStart; uint32 seqLen; uint32 seqLenMax = ~uint32ZERO; uint32 namePos; readBuffer ib(_filename); char x = ib.read(); #ifdef DEBUGINDEX fprintf(stderr, "readBuffer '%s' eof=%d x=%c %d\n", _filename, ib.eof(), x, x); #endif // Build it. // Skip whitespace at the start of the sequence. while ((ib.eof() == false) && (alphabet.isWhitespace(x) == true)) { #ifdef DEBUGINDEX fprintf(stderr, "skip '%c' %d\n", x, x); #endif x = ib.read(); } while (ib.eof() == false) { #ifdef DEBUGINDEX fprintf(stderr, "index\n"); #endif // We should be at a '@' character now. Fail if not. if (x != '@') fprintf(stderr, "fastqFile::constructIndex()-- ERROR3: In %s, expected '@' at beginning of defline, got '%c' instead.\n", _filename, x), exit(1); // Save info - ib's position is correctly at the first letter in // the defline (which might be whitespace), but the reader // expects our position to be at the '@' -- hence the -1. seqStart = ib.tell() - 1; seqLen = 0; namePos = namesLen; // Read that first letter x = ib.read(); // Copy the name to the names while ((ib.eof() == false) && (alphabet.isWhitespace(x) == false)) { if (namesLen + 1 >= namesMax) { namesMax += 32 * 1024 * 1024; char *nt = new char [namesMax]; memcpy(nt, _names, namesLen); delete [] _names; _names = nt; } _names[namesLen++] = x; #ifdef DEBUGINDEX fprintf(stderr, "name += %c\n", x); #endif x = ib.read(); } if (namesLen + 1 >= namesMax) { namesMax += 32 * 1024 * 1024; char *nt = new char [namesMax]; memcpy(nt, _names, namesLen); delete [] _names; _names = nt; } _names[namesLen++] = 0; // Skip the rest of the defline while ((ib.eof() == false) && (x != '\r') && (x != '\n')) { #ifdef DEBUGINDEX fprintf(stderr, "skip let %c\n", x); #endif x = ib.read(); } // Skip whitespace between the defline and the sequence. while ((ib.eof() == false) && (alphabet.isWhitespace(x) == true)) { #ifdef DEBUGINDEX fprintf(stderr, "skip num %d\n", x); #endif x = ib.read(); } #ifdef DEBUGINDEX fprintf(stderr, "x=%c peek=%c\n", x, ib.peek()); #endif // Count sequence length while ((ib.eof() == false) && (x != '+')) { #ifdef DEBUGINDEX fprintf(stderr, "seqlen %s %c\n", (alphabet.isWhitespace(x) == false) ? "save" : "skip", x); #endif if (alphabet.isWhitespace(x) == false) seqLen++; if (seqLen >= seqLenMax) fprintf(stderr, "fastqFile::constructIndex()-- ERROR: In %s, sequence '%s' is too long. Maximum length is %u bases.\n", _filename, _names + namePos, seqLenMax), exit(1); x = ib.read(); } // Save to the index. if (indexLen >= indexMax) { fprintf(stderr, "REALLOC len="F_U32" from "F_U32" to "F_U32"\n", indexLen, indexMax, indexMax * 2); indexMax *= 2; fastqFileIndex *et = new fastqFileIndex[indexMax]; memcpy(et, _index, sizeof(fastqFileIndex) * indexLen); delete [] _index; _index = et; } _index[indexLen]._seqPosition = seqStart; _index[indexLen]._seqLength = seqLen; #if 0 if ((indexLen * sizeof(fastqFileIndex) > 131000) && (indexLen * sizeof(fastqFileIndex) < 131200)) fprintf(stderr, "INDEX pos="F_U64" iid="F_U32" len="F_U32" pos="F_U64"\n", indexLen * sizeof(fastqFileIndex), indexLen, seqLen, seqStart); #endif indexLen++; // Skip the rest of the QV def line, then the entire QV line, then load the '@' for the next sequence. //x = ib.read(); assert((ib.eof() == true) || (x == '+')); while ((ib.eof() == false) && (x != '\r') && (x != '\n')) x = ib.read(); x = ib.read(); while ((ib.eof() == false) && (x != '\r') && (x != '\n')) x = ib.read(); while ((ib.eof() == false) && (x != '@')) x = ib.read(); } // Fill out the index meta data struct stat fastqstat; errno = 0; if (stat(_filename, &fastqstat)) fprintf(stderr, "fastqFile::constructIndex()-- stat() of file '%s' failed: %s\n", _filename, strerror(errno)), exit(1); _header._magic[0] = FASTQ_MAGICNUMBER1; _header._magic[1] = FASTQ_MAGICNUMBER2; _header._numberOfSequences = indexLen; _header._namesLength = namesLen; _header._fastqFileSize = fastqstat.st_size; _header._fastqModificationTime = fastqstat.st_mtime; _header._fastqCreationTime = fastqstat.st_ctime; // Dump the index, if possible. errno = 0; FILE *I = fopen(indexname, "w"); if (errno) return; fwrite(&_header, sizeof(fastqFileHeader), 1, I); fwrite( _index, sizeof(fastqFileIndex), _header._numberOfSequences, I); fwrite( _names, sizeof(char), _header._namesLength, I); fclose(I); } canu-1.3/src/meryl/libleaff/fastqFile.H000066400000000000000000000051531272335422300200240ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef FASTQFILE_H #define FASTQFILE_H #include "seqFile.H" struct fastqFileHeader { uint64 _magic[2]; uint32 _numberOfSequences; // Number of sequences in the file uint32 _namesLength; // Bytes in the names uint64 _fastqFileSize; // st_size - size of file in bytes uint64 _fastqModificationTime; // st_mtime - time of last data modification uint64 _fastqCreationTime; // st_ctime - time of last file status change }; struct fastqFileIndex { uint64 _seqPosition; // Position of the sequence in the file uint32 _seqLength; // Length of the sequence (no whitespace counted) }; class fastqFile : public seqFile { protected: fastqFile(const char *filename); fastqFile(); public: ~fastqFile(); protected: seqFile *openFile(const char *filename); public: uint32 find(const char *sequencename); uint32 getSequenceLength(uint32 iid); bool getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax); bool getSequence(uint32 iid, uint32 bgn, uint32 end, char *s); private: void clear(void); void loadIndex(char *indexname); void constructIndex(void); readBuffer *_rb; fastqFileHeader _header; fastqFileIndex *_index; char *_names; uint32 _nextID; // Next sequence in the read buffer uint32 _gs_iid; uint32 _gs_pos; friend class seqFactory; }; #endif // FASTQFILE_H canu-1.3/src/meryl/libleaff/fastqStdin.C000066400000000000000000000162511272335422300202220ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "fastqStdin.H" #include "dnaAlphabets.H" fastqStdin::fastqStdin(const char *filename) { clear(); #ifdef DEBUG fprintf(stderr, "fastqStdin::fastqStdin()-- '%s'\n", (filename) ? filename : "NULLPOINTER"); #endif if (filename == 0L) { strcpy(_filename, "(stdin)"); _rb = new readBuffer("-"); } else { _pipe = popen(filename, "r"); _rb = new readBuffer(_pipe); } } fastqStdin::fastqStdin() { clear(); } fastqStdin::~fastqStdin() { delete _rb; delete [] _header; delete [] _sequence; } seqFile * fastqStdin::openFile(const char *filename) { #ifdef DEBUG fprintf(stderr, "fastqStdin::openFile()-- '%s'\n", (filename) ? filename : "NULLPOINTER"); #endif if (((filename == 0L) && (isatty(fileno(stdin)) == 0)) || ((filename != 0L) && (filename[0] == '-') && (filename[1] == 0))) return(new fastqStdin(0L)); // The stdin variants also handle compressed inputs (because we can't seek in these). if (filename == 0L) return(0L); uint32 fl = strlen(filename); char cmd[32 + fl]; if ((filename[fl-3] == '.') && (filename[fl-2] == 'g') && (filename[fl-1] == 'z')) sprintf(cmd, "gzip -dc %s", filename); else if ((filename[fl-4] == '.') && (filename[fl-3] == 'b') && (filename[fl-2] == 'z') && (filename[fl-1] == '2')) sprintf(cmd, "bzip2 -dc %s", filename); else if ((filename[fl-3] == '.') && (filename[fl-2] == 'x') && (filename[fl-1] == 'z')) sprintf(cmd, "xz -dc %s", filename); else return(0L); return(new fastqStdin(cmd)); } uint32 fastqStdin::getNumberOfSequences(void) { if (_rb->peek() == 0) return(_nextIID); else return(_nextIID + 1); } uint32 fastqStdin::find(const char *sequencename) { fprintf(stderr, "fastqStdin::find()-- ERROR! Used for random access on sequence '%s'.\n", sequencename); assert(0); return(~uint32ZERO); } uint32 fastqStdin::getSequenceLength(uint32 iid) { if (iid == _nextIID) if (loadNextSequence(_header, _headerLen, _headerMax, _sequence, _sequenceLen, _sequenceMax) == false) return(0); if (iid + 1 != _nextIID) { fprintf(stderr, "fastqStdin::getSequence()-- ERROR! Used for random access. Requested iid=%u, at iid=%u\n", iid, _nextIID); assert(0); } return(strlen(_sequence)); } bool fastqStdin::getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax) { bool ret = true; #ifdef DEBUG fprintf(stderr, "fastqStdin::getSequence(full)-- "F_U32"\n", iid); #endif if (iid == _nextIID) if (loadNextSequence(_header, _headerLen, _headerMax, _sequence, _sequenceLen, _sequenceMax) == false) return(false); if (iid + 1 != _nextIID) { fprintf(stderr, "fastqStdin::getSequence(full)-- ERROR! Used for random access. Requested iid=%u, at iid=%u\n", iid, _nextIID); assert(0); } if (hLen < _headerMax) { delete [] h; hMax = _headerMax; h = new char [hMax]; } if (sLen < _sequenceMax) { delete [] s; sMax = _sequenceMax; s = new char [sMax]; } memcpy(h, _header, _headerLen + 1); hLen = _headerLen; memcpy(s, _sequence, _sequenceLen + 1); sLen = _sequenceLen; return(true); } bool fastqStdin::getSequence(uint32 iid, uint32 bgn, uint32 end, char *s) { fprintf(stderr, "fastqStdin::getSequence(part)-- ERROR! Used for random access on iid "F_U32" from position "F_U32"-"F_U32".\n", iid, bgn, end); assert(0); return(false); } void fastqStdin::clear(void) { memset(_filename, 0, FILENAME_MAX); memset(_typename, 0, FILENAME_MAX); _randomAccessSupported = false; strcpy(_typename, "FastQstream"); _numberOfSequences = ~uint32ZERO; _rb = 0L; _nextIID = 0; _pipe = 0L; _header = 0L; _headerLen = 0; _headerMax = 0; _sequence = 0L; _sequenceLen = 0; _sequenceMax = 0; } bool fastqStdin::loadNextSequence(char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax) { if (hMax == 0) { hMax = 2048; h = new char [hMax]; } if (sMax == 0) { sMax = 2048; s = new char [sMax]; } hLen = 0; sLen = 0; char x = _rb->read(); // Skip whitespace at the start of the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // We should be at a '@' character now. Fail if not. if (_rb->eof() == true) return(false); if (x != '@') fprintf(stderr, "fastqStdin::loadNextSequence(part)-- ERROR: In %s, expected '@' at beginning of defline, got '%c' instead.\n", _filename, x), exit(1); // Skip the '@' in the defline x = _rb->read(); // Skip whitespace between the '@' and the defline while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true) && (x != '\r') && (x != '\n')) x = _rb->read(); // Copy the defline, until the first newline. while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) { h[hLen++] = x; if (hLen >= hMax) { //fprintf(stderr, "realloc header\n"); hMax += 2048; char *H = new char [hMax]; memcpy(H, h, hLen); delete [] h; h = H; } x = _rb->read(); } h[hLen] = 0; // Skip whitespace between the defline and the sequence. while ((_rb->eof() == false) && (alphabet.isWhitespace(x) == true)) x = _rb->read(); // Copy the sequence, until EOF or the start of the QV bases. while ((_rb->eof() == false) && (_rb->peek() != '+')) { if (alphabet.isWhitespace(x) == false) { s[sLen++] = x; if (sLen >= sMax) { //fprintf(stderr, "realloc sequence\n"); sMax *= 2; char *S = new char [sMax]; memcpy(S, s, sLen); delete [] s; s = S; } } x = _rb->read(); } s[sLen] = 0; // Skip the rest of the QV id line and then the entire QV line. //x = _rb->read(); assert((_rb->eof() == true) || (x == '+')); while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) x = _rb->read(); x = _rb->read(); while ((_rb->eof() == false) && (x != '\r') && (x != '\n')) x = _rb->read(); _nextIID++; return(true); } canu-1.3/src/meryl/libleaff/fastqStdin.H000066400000000000000000000043651272335422300202320ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef FASTQSTDIN_H #define FASTQSTDIN_H #include "seqFile.H" class fastqStdin : public seqFile { protected: fastqStdin(const char *filename); fastqStdin(); public: ~fastqStdin(); protected: seqFile *openFile(const char *filename); public: uint32 getNumberOfSequences(void); public: uint32 find(const char *sequencename); uint32 getSequenceLength(uint32 iid); bool getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax); bool getSequence(uint32 iid, uint32 bgn, uint32 end, char *s); private: void clear(void); bool loadNextSequence(char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax); readBuffer *_rb; uint32 _nextIID; FILE *_pipe; char *_header; uint32 _headerLen; uint32 _headerMax; char *_sequence; uint32 _sequenceLen; uint32 _sequenceMax; char *_quality; uint32 _qualityLen; uint32 _qualityMax; friend class seqFactory; }; #endif // FASTQSTDIN_H canu-1.3/src/meryl/libleaff/gkStoreFile.C000066400000000000000000000074231272335422300203210ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-FEB-04 to 2015-AUG-14 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "gkStoreFile.H" //#include "AS_UTL_fileIO.H" gkStoreFile::gkStoreFile() { clear(); } gkStoreFile::gkStoreFile(const char *name) { clear(); strcpy(_filename, name); gkp = gkStore::gkStore_open(_filename); _numberOfSequences = gkp->gkStore_getNumReads(); //fprintf(stderr, "Opened '%s' with %u reads\n", _filename, _numberOfSequences); } gkStoreFile::~gkStoreFile() { gkp->gkStore_close(); } seqFile * gkStoreFile::openFile(const char *name) { struct stat st; // Assume it's a gkStore if it is a directory, and the info / reads / blobs files exist. char infoName[FILENAME_MAX]; char readName[FILENAME_MAX]; char blobName[FILENAME_MAX]; sprintf(infoName, "%s/info", name); sprintf(readName, "%s/reads", name); sprintf(blobName, "%s/blobs", name); if ((AS_UTL_fileExists(name, true) == false) || (AS_UTL_fileExists(infoName) == false) || (AS_UTL_fileExists(readName) == false) || (AS_UTL_fileExists(blobName) == false)) return(0L); // Yup, probably a gkStore. If it isn't, the gkStore() constructor blows up. return(new gkStoreFile(name)); } bool gkStoreFile::getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax) { if (iid > _numberOfSequences) { fprintf(stderr, "gkStoreFile::getSequence()-- iid %u exceeds number in store %u\n", iid, _numberOfSequences); return(false); } iid++; uint32 rLength = gkp->gkStore_getRead(iid)->gkRead_sequenceLength(); if (hMax < 32) { delete h; h = new char [32]; hMax = 32; } if (sMax < rLength) { delete s; s = new char [rLength + 1]; sMax = rLength; } hLen = sprintf(h, F_U32, iid); sLen = rLength; gkp->gkStore_loadReadData(iid, &readData); memcpy(s, readData.gkReadData_getSequence(), sizeof(char) * rLength); s[sLen] = 0; return(true); } bool gkStoreFile::getSequence(uint32 iid, uint32 bgn, uint32 end, char *s) { if (iid > _numberOfSequences) { fprintf(stderr, "gkStoreFile::getSequence()-- iid %u exceeds number in store %u\n", iid, _numberOfSequences); return(false); } iid++; uint32 rLength = gkp->gkStore_getRead(iid)->gkRead_sequenceLength(); //fprintf(stderr, "return canu iid %u of length %u\n", iid, rLength); assert(bgn < end); assert(bgn <= rLength); assert(end <= rLength); gkp->gkStore_loadReadData(iid, &readData); memcpy(s, readData.gkReadData_getSequence() + bgn, sizeof(char) * (end - bgn)); s[end-bgn] = 0; return(true); } void gkStoreFile::clear(void) { memset(_filename, 0, FILENAME_MAX); memset(_typename, 0, FILENAME_MAX); strcpy(_typename, "GKSTORE"); _numberOfSequences = 0; } canu-1.3/src/meryl/libleaff/gkStoreFile.H000066400000000000000000000040461272335422300203240ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-FEB-04 to 2015-MAR-17 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef GKSTOREFILE_H #define GKSTOREFILE_H #include "seqFile.H" #include "gkStore.H" class gkStoreFile : public seqFile { protected: gkStoreFile(const char *filename); gkStoreFile(); public: ~gkStoreFile(); protected: seqFile *openFile(const char *name); public: uint32 find(const char *sequencename) { fprintf(stderr, "gkStoreFile::find()-- Lookup of sequencename '%s' not supported.\n", sequencename); assert(0); return(0); }; uint32 getSequenceLength(uint32 iid) { return(gkp->gkStore_getRead(iid + 1)->gkRead_sequenceLength()); }; bool getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax); bool getSequence(uint32 iid, uint32 bgn, uint32 end, char *s); private: void clear(void); gkStore *gkp; gkReadData readData; friend class seqFactory; }; #endif // GKSTOREFILE_H canu-1.3/src/meryl/libleaff/merStream.C000066400000000000000000000044261272335422300200420ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "merStream.H" merStream::merStream(kMerBuilder *kb, seqStream *ss, bool kbown, bool ssown) { _kb = kb; _ss = ss; _kbdelete = kbown; _ssdelete = ssown; _beg = uint64ZERO; _end = ~uint64ZERO; _kb->clear(); _invalid = true; } merStream::~merStream() { if (_kbdelete) delete _kb; if (_ssdelete) delete _ss; } void merStream::rewind(void) { _ss->rewind(); _kb->clear(); _invalid = true; } void merStream::rebuild(void) { _ss->setPosition(_ss->strPos() - _kb->theFMer().getMerSpan()); _kb->clear(); _invalid = true; } void merStream::setBaseRange(uint64 beg, uint64 end) { assert(beg < end); //fprintf(stderr, "merStream::setBaseRange()-- from "uint64FMT" to "uint64FMT".\n", beg, end); // We can't tell the seqStore when to stop; while we could compute the span of a spaced seed, we // cannot compute it for a compressed seed. We need to stop iterating when the beginning of the // mer reaches the requested end. _ss->setRange(beg, ~uint64ZERO); _beg = beg; _end = end; _kb->clear(); _invalid = true; } uint64 merStream::approximateNumberOfMers(void) { uint64 approx = _end - _beg; uint64 k = _kb->merSize(); // If we don't know the range, sum all the sequence lengths, otherwise, it's just the length from // begin to end. if (_end == ~uint64ZERO) { approx = uint64ZERO; for (uint32 s=0; s<_ss->numberOfSequences(); s++) { uint32 l = _ss->lengthOf(s); if (l > k) approx += l - k + 1; } } return(approx); } canu-1.3/src/meryl/libleaff/merStream.H000066400000000000000000000076611272335422300200530ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef MERSTREAM_H #define MERSTREAM_H #include "seqFile.H" #include "seqStream.H" #include "kMer.H" // // merStream needs exclusive use of a kMerBuilder and a seqStream. // // The kMerBuilder can be used over and over. I think snapper is the // only one that does this though. // // The seqStream can be used elsewhere, but ONLY for looking up // positions. // // The merStream does NOT assume ownership of either of these, unless // the own flags are set. // // The stream is not valid until nextMer is called; allowing loops of // while (MS->nextMer()) { // process(MS->theFMer()); // } // // setRange() positions refer to ACGT letters in the input, NOT mers. // rewind() repositions the file to the start of the range. // class merStream { public: merStream(kMerBuilder *kb, seqStream *ss, bool kbown=false, bool ssown=false); ~merStream(); kMer const & theFMer(void) { assert(_invalid == false); return(_kb->theFMer()); }; kMer const & theRMer(void) { assert(_invalid == false); return(_kb->theRMer()); }; kMer const & theCMer(void) { assert(_invalid == false); return(_kb->theCMer()); }; bool nextMer(uint32 skip=0) { char ch; do { ch = _ss->get(); if (ch == 0) return(false); } while ((_kb->addBase(ch) == true) || (skip-- > 0)); _kb->mask(); _invalid = false; #if 0 char merstring[256]; fprintf(stderr, "merStream::nextMer()-- seqPos="uint64FMT" merPos="uint64FMT" span="uint32FMT" base0span="uint32FMT" end="uint64FMT" %s %s\n", _ss->strPos(), _ss->strPos() - theFMer().getMerSpan(), theFMer().getMerSpan(), _kb->baseSpan(0), _end, _kb->theFMer().merToString(merstring), (_ss->strPos() - theFMer().getMerSpan() < _end) ? "" : "STOP"); #endif // The mer is out of range if: // o it begins at or past the _end // o the span of the first base ends at or past the _end // // If the mer isn't spaced, the base span is always 1. If it is spaced, the span will be // between 1 and ... who knows. return(_ss->strPos() - theFMer().getMerSpan() + _kb->baseSpan(0) - 1 < _end); }; void rewind(void); void rebuild(void); void setBaseRange(uint64 beg, uint64 end); uint64 thePositionInSequence(void) { assert(_invalid == false); return(_ss->seqPos() - theFMer().getMerSpan()); }; uint64 thePositionInStream(void) { assert(_invalid == false); return(_ss->strPos() - theFMer().getMerSpan()); }; uint64 theSequenceNumber(void) { assert(_invalid == false); return(_ss->seqIID()); }; uint64 approximateNumberOfMers(void); private: kMerBuilder *_kb; seqStream *_ss; bool _kbdelete; bool _ssdelete; bool _invalid; uint64 _beg; uint64 _end; }; #endif // MERSTREAM_H canu-1.3/src/meryl/libleaff/selftest.C000066400000000000000000000042541272335422300177330ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ { seqFile *SF = openSeqFile(argv[1]); fprintf(stdout, "source '%s' of type '%s' has "uint32FMT" sequences.\n", SF->getSourceName(), SF->getFileTypeName(), SF->getNumberOfSequences()); fprintf(stdout, "getSequenceLength() vs getSequence(full)\n"); { char *h = 0L; char *s = 0L; uint32 hLen=0, hMax=0; uint32 sLen=0, sMax=0; for (uint32 sid=0; sidgetNumberOfSequences(); sid++) { SF->getSequence(sid, h, hLen, hMax, s, sLen, sMax); if ((strlen(s) != SF->getSequenceLength(sid)) || (strlen(s) != sLen) || (SF->getSequenceLength(sid) != sLen)) { fprintf(stdout, "length differ for sid="uint32FMT" h='%s' strlen(s)=%d sLen="uint32FMT" getSequenceLength()="uint32FMT"\n", sid, h, strlen(s), sLen, SF->getSequenceLength(sid)); } } delete [] h; delete [] s; } fprintf(stdout, "getSequenceLength() vs getSequence(part)\n"); { char *p = new char [128 * 1024 * 1024]; for (uint32 sid=0; sidgetNumberOfSequences(); sid++) { SF->getSequence(sid, 0, SF->getSequenceLength(sid), p); if (strlen(p) != SF->getSequenceLength(sid)) { fprintf(stdout, "length differ for sid="uint32FMT" strlen(s)=%d getSequenceLength()="uint32FMT"\n", sid, strlen(p), SF->getSequenceLength(sid)); } } delete [] p; } return(0); } canu-1.3/src/meryl/libleaff/seqCache.C000066400000000000000000000114551272335422300176170ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2014-DEC-08 to 2014-DEC-22 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "seqCache.H" #include "seqFactory.H" seqCache::seqCache(const char *filename, uint32 cachesize, bool verbose) { _fb = openSeqFile(filename); _idToGetNext = 0; _allSequencesLoaded = false; _reportLoading = verbose; _cacheMap = 0L; _cacheSize = 0; _cacheNext = 0; _cache = 0L; setCacheSize(cachesize); } seqCache::~seqCache() { flushCache(); delete _fb; delete [] _cacheMap; delete [] _cache; } uint32 seqCache::getSequenceIID(char *name) { uint32 iid = ~uint32ZERO; // If the name is all integers, AND below the number of sequences // we have, return that, otherwise, look it up. // bool isInt = true; char *x = name; while (*x) { if ((*x < '0') || ('9' < *x)) isInt = false; x++; } if (isInt) iid = strtouint32(name); if (iid >= _fb->getNumberOfSequences()) iid = _fb->find(name); #ifdef DEBUG fprintf(stderr, "seqCache::getSequenceIID()-- '%s' -> "F_U32"\n", name, iid); #endif return(iid); } seqInCore * seqCache::getSequenceInCore(uint32 iid) { uint32 cacheID = ~uint32ZERO; seqInCore *retSeq = 0L; if ((_fb->randomAccessSupported() == true) && (iid >= _fb->getNumberOfSequences())) return(0L); if (_allSequencesLoaded == true) { cacheID = iid; } else if ((_cacheSize > 0) && (_cacheMap[iid] != ~uint32ZERO)) { cacheID = _cacheMap[iid]; } else { uint32 hLen=0, hMax=0, sLen=0, sMax=0; char *h=0L, *s=0L; if (_fb->getSequence(iid, h, hLen, hMax, s, sLen, sMax) == false) return(0L); retSeq = new seqInCore(iid, h, hLen, s, sLen, true); // Remove any old cached sequence, then store the one we just made if (_cache) { if (_cache[_cacheNext]) { _cacheMap[_cache[_cacheNext]->getIID()] = ~uint32ZERO; delete _cache[_cacheNext]; } _cache[_cacheNext] = retSeq; _cacheMap[iid] = _cacheNext; cacheID = _cacheNext; retSeq = 0L; _cacheNext = (_cacheNext + 1) % _cacheSize; } } // If no retSeq set, make a copy of the one we have in the cache. if ((retSeq == 0L) && (cacheID != ~uint32ZERO)) retSeq = new seqInCore(iid, _cache[cacheID]->header(), _cache[cacheID]->headerLength(), _cache[cacheID]->sequence(), _cache[cacheID]->sequenceLength(), false); return(retSeq); } void seqCache::setCacheSize(uint32 cachesize) { uint32 ns = _fb->getNumberOfSequences(); flushCache(); if (cachesize == 0) { _cacheMap = 0L; _cacheSize = 0; _cacheNext = 0; _cache = 0L; return; } _cacheMap = new uint32 [ns]; _cacheSize = cachesize; _cacheNext = 0; _cache = new seqInCore * [_cacheSize]; for (uint32 i=0; igetNumberOfSequences(); _cacheNext = 0; _cache = new seqInCore * [_cacheSize]; for (uint32 iid=0; iid<_cacheSize; iid++) { uint32 hLen=0, hMax=0, sLen=0, sMax=0; char *h=0L, *s=0L; if (_fb->getSequence(iid, h, hLen, hMax, s, sLen, sMax) == false) fprintf(stderr, "seqCache::loadAllSequences()-- Failed to load iid "F_U32".\n", iid), exit(1); _cache[iid] = new seqInCore(iid, h, hLen, s, sLen, true); } _allSequencesLoaded = true; } void seqCache::flushCache(void) { if (_fb == 0L) return; if (_cacheMap) { uint32 ns = _fb->getNumberOfSequences(); for (uint32 i=0; igetSourceName()); }; const char *getFileTypeName(void) { return(_fb->getFileTypeName()); }; bool randomAccessSupported(void) { return(_fb->randomAccessSupported()); }; uint32 getNumberOfSequences(void) { return(_fb->getNumberOfSequences()); }; uint32 getSequenceLength(uint32 iid) { return(_fb->getSequenceLength(iid)); }; void setCacheSize(uint32 cachesize); void loadAllSequences(void); void flushCache(void); private: seqFile *_fb; uint32 _idToGetNext; bool _allSequencesLoaded; bool _reportLoading; uint32 *_cacheMap; // Maps ID to cache entry uint32 _cacheSize; // Size of cache uint32 _cacheNext; // Next cache spot to use seqInCore **_cache; // Cache of sequences }; #endif // SEQCACHE_H canu-1.3/src/meryl/libleaff/seqFactory.C000066400000000000000000000043161272335422300202210ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2015-FEB-04 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "seqFactory.H" #include "fastaFile.H" #include "fastaStdin.H" #include "fastqFile.H" #include "fastqStdin.H" #include "seqStore.H" #include "gkStoreFile.H" seqFactory *seqFactory::me = 0L; seqFactory::seqFactory() { _filesNum = 0; _filesMax = 16; _files = new seqFile * [_filesMax]; registerFile(new fastaFile); registerFile(new fastaStdin); registerFile(new fastqFile); registerFile(new fastqStdin); registerFile(new seqStore); registerFile(new gkStoreFile); //registerFile(new sffFile); } seqFactory::~seqFactory() { for (uint32 i=0; i<_filesNum; i++) delete _files[i]; delete [] _files; } void seqFactory::registerFile(seqFile *f) { if (_filesNum >= _filesMax) { fprintf(stderr, "seqFactory::registerFile()-- Wow! You registered lots of files! Now fix %s at line %d.\n", __FILE__, __LINE__); exit(1); } _files[_filesNum++] = f; } seqFile * seqFactory::openFile(const char *name) { seqFile *n = 0L; for (uint32 i=0; i<_filesNum; i++) { n = _files[i]->openFile(name); if (n) return(n); } fprintf(stderr, "seqFactory::registerFile()-- Cannot determine type of file '%s'. Tried:\n", name); for (uint32 i=0; i<_filesNum; i++) fprintf(stderr, "seqFactory::registerFile()-- '%s'\n", _files[i]->getFileTypeName()); exit(1); return(n); } canu-1.3/src/meryl/libleaff/seqFactory.H000066400000000000000000000026571272335422300202340ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef SEQFACTORY_H #define SEQFACTORY_H #include "seqFile.H" class seqFactory { protected: seqFactory(); ~seqFactory(); public: static seqFactory *instance(void) { if (me == 0L) me = new seqFactory; return(me); }; void registerFile(seqFile *f); seqFile *openFile(const char *name); private: static seqFactory *me; uint32 _filesNum; uint32 _filesMax; seqFile **_files; }; #define openSeqFile(S) seqFactory::instance()->openFile((S)) #endif // SEQFACTORY_H canu-1.3/src/meryl/libleaff/seqFile.H000066400000000000000000000050741272335422300175000ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef SEQFILE_H #define SEQFILE_H #undef DEBUG #include "AS_global.H" #include "readBuffer.H" // General flow of the constructors is: // Clear all data // Open the file // Set _filename, _typename // Read/build the index structure // Position the file to the first read // Set _numberOfSequences (IMPORTANT, and subtle) class seqFile { protected: seqFile(const char *UNUSED(filename)) {}; seqFile() {}; public: virtual ~seqFile() {}; protected: virtual seqFile *openFile(const char *filename) = 0; public: virtual const char *getSourceName(void) { return(_filename); }; virtual const char *getFileTypeName(void) { return(_typename); }; virtual bool randomAccessSupported(void) { return(_randomAccessSupported); }; virtual uint32 getNumberOfSequences(void) { return(_numberOfSequences); }; public: virtual uint32 find(const char *sequencename) = 0; virtual uint32 getSequenceLength(uint32 id) = 0; virtual bool getSequence(uint32 id, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax) = 0; virtual bool getSequence(uint32 iid, uint32 bgn, uint32 end, char *s) = 0; protected: char _filename[FILENAME_MAX]; char _typename[FILENAME_MAX]; bool _randomAccessSupported; uint32 _numberOfSequences; friend class seqFactory; }; #endif // SEQFILE_H canu-1.3/src/meryl/libleaff/seqStore.C000066400000000000000000000425571272335422300177170ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "seqStore.H" #include "seqCache.H" #include "dnaAlphabets.H" #include "speedCounter.H" // Says 'kmerSeqStoreFile' #define SEQSTORE_MAGICNUMBER1 0x5371655372656d6bULL #define SEQSTORE_MAGICNUMBER2 0x656c694665726f74ULL seqStore::seqStore(const char *filename) { clear(); strcpy(_filename, filename); errno = 0; FILE *F = fopen(_filename, "r"); if (errno) fprintf(stderr, "seqStore::seqStore()-- Failed to open '%s': %s\n", _filename, strerror(errno)), exit(1); fread(&_header, sizeof(seqStoreHeader), 1, F); fclose(F); //_indexBPF = new bitPackedFile(_filename, _header._indexStart); //_blockBPF = new bitPackedFile(_filename, _header._blockStart); //_namesBPF = new bitPackedFile(_filename, _header._namesStart); _bpf = new bitPackedFile(_filename, sizeof(seqStoreHeader)); _numberOfSequences = _header._numberOfSequences; } seqStore::seqStore() { clear(); } seqStore::~seqStore() { //if ((_filename) && (_filename[0] != 0)) // fprintf(stderr, "Closing seqStore '%s'\n", _filename); delete _bpf; delete [] _index; delete [] _block; delete [] _names; delete _indexBPF; delete _blockBPF; delete _namesBPF; } seqFile * seqStore::openFile(const char *filename) { uint64 magic1, magic2; struct stat st; errno = 0; stat(filename, &st); if (errno) return(0L); if ((st.st_mode & S_IFREG) == 0) return(0L); // Check the magic. Fail if not correct. errno = 0; FILE *F = fopen(filename, "r"); if (errno) return(0L); fread(&magic1, sizeof(uint64), 1, F); fread(&magic2, sizeof(uint64), 1, F); fclose(F); if ((magic1 != SEQSTORE_MAGICNUMBER1) || (magic2 != SEQSTORE_MAGICNUMBER2)) return(0L); return(new seqStore(filename)); } // If this proves far too slow, rewrite the _names string to separate IDs with 0xff, then use // strstr on the whole thing. To find the ID, scan down the string counting the number of 0xff's. // // Similar code is used for fastaFile::find() // uint32 seqStore::find(const char *sequencename) { if (_names == NULL) loadIndex(); char *ptr = _names; for (uint32 iid=0; iid < _header._numberOfSequences; iid++) { if (strcmp(sequencename, ptr) == 0) return(iid); while (*ptr) ptr++; ptr++; } return(~uint32ZERO); } uint32 seqStore::getSequenceLength(uint32 iid) { if (_index == NULL) loadIndex(); return((iid < _header._numberOfSequences) ? _index[iid]._seqLength : 0); } bool seqStore::getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax) { if (_index == NULL) loadIndex(); if (iid >= _header._numberOfSequences) { fprintf(stderr, "seqStore::getSequence(full)-- iid "F_U32" more than number of sequences "F_U32"\n", iid, _header._numberOfSequences); return(false); } if (sMax == 0) s = 0L; // So the delete below doesn't bomb if (hMax == 0) h = 0L; if (sMax < _index[iid]._seqLength + 1) { sMax = _index[iid]._seqLength + 1024; delete [] s; s = new char [sMax]; } if (hMax < _index[iid]._hdrLength + 1) { hMax = _index[iid]._hdrLength + 1024; delete [] h; h = new char [hMax]; } hLen = 0; sLen = 0; // Copy the defline into h memcpy(h, _names + _index[iid]._hdrPosition, _index[iid]._hdrLength); h[_index[iid]._hdrLength] = 0; // Decode and copy the sequence into s uint32 seqLen = _index[iid]._seqLength; uint32 block = _index[iid]._block; uint64 seekpos = _index[iid]._seqPosition * 2; _bpf->seek(seekpos); while (sLen < seqLen) { assert(_bpf->tell() == _block[block]._bpf * 2); assert(sLen == _block[block]._pos); if (_block[block]._isACGT == 0) { memset(s + sLen, 'N', _block[block]._len); sLen += _block[block]._len; } else { for (uint32 xx=0; xx<_block[block]._len; xx++) { s[sLen++] = alphabet.bitsToLetter(_bpf->getBits(2)); } } block++; } s[sLen] = 0; return(true); } bool seqStore::getSequence(uint32 iid, uint32 bgn, uint32 end, char *s) { if (_index == NULL) loadIndex(); if (iid >= _header._numberOfSequences) { fprintf(stderr, "seqStore::getSequence(part)-- iid "F_U32" more than number of sequences "F_U32"\n", iid, _header._numberOfSequences); return(false); } if (bgn >= end) { fprintf(stderr, "seqStore::getSequence(part)-- for iid "F_U32"; invalid bgn="F_U32" end="F_U32"; seqLen="F_U32"\n", iid, bgn, end, _index[iid]._seqLength); return(false); } // Decode and copy the sequence into s uint32 block = _index[iid]._block; uint32 sLen = 0; // length of sequence we've copied uint32 sPos = 0; // position in the sequence // Skip blocks before we care. // while (sPos + _block[block]._len < bgn) { sPos += _block[block]._len; block++; } assert(sPos == _block[block]._pos); // Move into the block (we could just set sPos = bgn...). sPos += bgn - _block[block]._pos; // Handle the partial block. Copy what is left in the block, or // the requested size, whichever is smaller. uint32 partLen = MIN((_block[block]._pos + _block[block]._len - bgn), (end - bgn)); if (_block[block]._isACGT == 0) { memset(s, 'N', partLen); sLen += partLen; _bpf->seek(_block[block+1]._bpf * 2); } else { _bpf->seek((_block[block]._bpf + bgn - _block[block]._pos) * 2); for (uint32 xx=0; xxgetBits(2)); } sPos += partLen; block++; while (sPos < end) { assert(_bpf->tell() == _block[block]._bpf * 2); assert(sPos == _block[block]._pos); // Like the partial block above, pick how much to copy as the // smaller of the block size and what is left to fill. partLen = MIN((_block[block]._len), (end - sPos)); if (_block[block]._isACGT == 0) { memset(s + sLen, 'N', partLen); sLen += partLen; } else { for (uint32 xx=0; xxgetBits(2)); } sPos += partLen; block++; } s[sLen] = 0; return(true); } void seqStore::clear(void) { memset(_filename, 0, FILENAME_MAX); memset(_typename, 0, FILENAME_MAX); strcpy(_typename, "seqStore"); _numberOfSequences = 0; _bpf = 0L; memset(&_header, 0, sizeof(seqStoreHeader)); _index = 0L; _block = 0L; _names = 0L; _indexBPF = 0L; _blockBPF = 0L; _namesBPF = 0L; _lastIIDloaded = ~uint32ZERO; } void seqStore::loadIndex(void) { if (_index) return; delete _indexBPF; _indexBPF = 0L; delete _blockBPF; _blockBPF = 0L; delete _namesBPF; _namesBPF = 0L; errno = 0; FILE *F = fopen(_filename, "r"); if (errno) fprintf(stderr, "seqStore::seqStore()-- Failed to open '%s': %s\n", _filename, strerror(errno)), exit(1); fread(&_header, sizeof(seqStoreHeader), 1, F); //fprintf(stderr, "seqStore::seqStore()-- Allocating space for "F_U32" sequences ("F_U64"MB)\n", _header._numberOfSequences, _header._numberOfSequences * sizeof(seqStoreIndex) / 1024 / 1024); //fprintf(stderr, "seqStore::seqStore()-- Allocating space for "F_U32" blocks ("F_U64"MB)\n", _header._numberOfBlocks, _header._numberOfBlocks * sizeof(seqStoreBlock) / 1024 / 1024); //fprintf(stderr, "seqStore::seqStore()-- Allocating space for "F_U32" labels ("F_U64"MB)\n", _header._namesLength, _header._namesLength * sizeof(char) / 1024 / 1024); _index = new seqStoreIndex [_header._numberOfSequences]; _block = new seqStoreBlock [_header._numberOfBlocks]; _names = new char [_header._namesLength]; fseeko(F, _header._indexStart, SEEK_SET); fread( _index, sizeof(seqStoreIndex), _header._numberOfSequences, F); #if 0 for (uint32 i=0; i<_header._numberOfSequences; i++) fprintf(stderr, "IDX[%4u] hdrPos=%u hdrLen=%u seqPos=%llu seqLen=%u block=%u\n", i, _index[i]._hdrPosition, _index[i]._hdrLength, _index[i]._seqPosition, _index[i]._seqLength, _index[i]._block); #endif fseeko(F, _header._blockStart, SEEK_SET); fread( _block, sizeof(seqStoreBlock), _header._numberOfBlocks, F); fseeko(F, _header._namesStart, SEEK_SET); fread( _names, sizeof(char), _header._namesLength, F); if (errno) fprintf(stderr, "seqStore::seqStore()-- Failed to read index from '%s': %s\n", _filename, strerror(errno)), exit(1); fclose(F); } static void addSeqStoreBlock(uint32 &BLOKmax, uint32 &BLOKlen, seqStoreBlock* &BLOK, seqStoreBlock &b, uint32 &nBlockACGT, uint32 &nBlockGAP, uint64 &nACGT) { //fprintf(stderr, "addSeqStoreBlock()-- BLOK max=%u len=%u ACGT=%u GAP=%u nACGT=%lu\n", // BLOKmax, BLOKlen, nBlockACGT, nBlockGAP, nACGT); if (b._len == 0) return; if (b._isACGT == 1) { nBlockACGT++; nACGT += b._len; } else { nBlockGAP++; } BLOK[BLOKlen++] = b; if (BLOKlen >= BLOKmax) { BLOKmax *= 2; seqStoreBlock *nb = new seqStoreBlock [BLOKmax]; memcpy(nb, BLOK, BLOKlen * sizeof(seqStoreBlock)); delete [] BLOK; BLOK = nb; } } void constructSeqStore(char *filename, seqCache *inputseq) { fprintf(stderr, "constructSeqStore()-- constructing seqStore '%s' from seqCache '%s' of type '%s'.\n", filename, inputseq->getSourceName(), inputseq->getFileTypeName()); seqStoreHeader HEAD; memset(&HEAD, sizeof(seqStoreHeader), 0); bitPackedFile *DATA = new bitPackedFile(filename, sizeof(seqStoreHeader), true); uint32 INDXmax = 1048576; seqStoreIndex *INDX = new seqStoreIndex [INDXmax]; uint32 BLOKmax = 1048576; uint32 BLOKlen = 0; seqStoreBlock *BLOK = new seqStoreBlock [BLOKmax]; uint32 NAMEmax = 32 * 1024 * 1024; uint32 NAMElen = 0; char *NAME = new char [NAMEmax]; seqInCore *sic = inputseq->getSequenceInCore(); uint64 nACGT = 0; uint32 nBlockACGT = 0; uint32 nBlockGAP = 0; uint32 nSequences = 0; speedCounter C(" reading sequences %7.0f sequences -- %5.0f sequences/second\r", 1.0, 0x1ffff, true); while (sic != NULL) { if (sic->sequence()) { char *seq = sic->sequence(); seqStoreBlock b; if (nSequences >= INDXmax) { seqStoreIndex *I = new seqStoreIndex[INDXmax * 2]; memcpy(I, INDX, sizeof(seqStoreIndex) * nSequences); delete [] INDX; INDXmax *= 2; INDX = I; } INDX[nSequences]._hdrPosition = NAMElen; INDX[nSequences]._hdrLength = sic->headerLength(); INDX[nSequences]._seqPosition = DATA->tell() / 2; INDX[nSequences]._seqLength = sic->sequenceLength(); INDX[nSequences]._block = BLOKlen; #if 0 fprintf(stderr, "ADD SEQUENCE hdr pos=%u len=%u seq pos=%u len=%u blok=%u\n", INDX[nSequences]._hdrPosition, INDX[nSequences]._hdrLength, INDX[nSequences]._seqPosition, INDX[nSequences]._seqLength, INDX[nSequences]._block); #endif #if SEQSTOREBLOCK_MAXPOS < uint64MASK(32) if (sic->sequenceLength() > SEQSTOREBLOCK_MAXPOS) fprintf(stderr, "constructSeqStore()-- sequence %s too long, must be shorter than "F_U64" Gbp.\n", sic->header(), SEQSTOREBLOCK_MAXPOS / 1024 / 1024 / 1024), exit(1); #endif #if SEQSTOREBLOCK_MAXIID < uint64MASK(32) if (sic->getIID() > SEQSTOREBLOCK_MAXPOS) fprintf(stderr, "constructSeqStore()-- too many sequences, must be fewer than "F_U64".\n", SEQSTOREBLOCK_MAXIID), exit(1); #endif if (NAMElen + sic->headerLength() + 1 > NAMEmax) { NAMEmax += 32 * 1024 * 1024; char *nm = new char [NAMEmax]; memcpy(nm, NAME, sizeof(char) * NAMElen); delete [] NAME; NAME = nm; } strcpy(NAME + NAMElen, sic->header()); NAMElen += sic->headerLength() + 1; b._isACGT = 0; b._iid = sic->getIID(); b._pos = 0; b._len = 0; b._bpf = DATA->tell() / 2; for (uint32 p=0; psequenceLength(); p++) { uint64 bits = alphabet.letterToBits(seq[p]); // If the length of the current block is too big (which would // soon overflow the bit field storing length) write out a // block and reset the length. // if (b._len == SEQSTOREBLOCK_MAXLEN) { addSeqStoreBlock(BLOKmax, BLOKlen, BLOK, b, nBlockACGT, nBlockGAP, nACGT); b._pos = p; b._len = 0; b._bpf = DATA->tell() / 2; } if (bits == 0xff) { // This letter is NOT ACGT. If the current block is an ACGT block, write it // and reset. // if (b._isACGT == 1) { addSeqStoreBlock(BLOKmax, BLOKlen, BLOK, b, nBlockACGT, nBlockGAP, nACGT); b._isACGT = 0; b._iid = sic->getIID(); b._pos = p; b._len = 0; b._bpf = DATA->tell() / 2; } } else { // This letter is ACGT. If the current block is NOT an ACGT block, write it // and reset. // if (b._isACGT == 0) { addSeqStoreBlock(BLOKmax, BLOKlen, BLOK, b, nBlockACGT, nBlockGAP, nACGT); b._isACGT = 1; b._iid = sic->getIID(); b._pos = p; b._len = 0; b._bpf = DATA->tell() / 2; } } // Always add one to the length of the current block, and // write out the base if the letter is ACGT. // b._len++; if (bits != 0xff) DATA->putBits(bits, 2); } // Emit the last block // addSeqStoreBlock(BLOKmax, BLOKlen, BLOK, b, nBlockACGT, nBlockGAP, nACGT); } // If there is no sequence, the index record for this sequence is left blank. // nSequences++; C.tick(); delete sic; sic = inputseq->getSequenceInCore(); } // And a sentinel EOF block -- gets the last position in the file, // useful for the binary search. We always have a space block at // the end of the list, but we don't care if we just used the last // block (and so we don't bother to reallocate the array if it is // full). BLOK[BLOKlen]._isACGT = 0; BLOK[BLOKlen]._iid = uint32MASK(32); BLOK[BLOKlen]._pos = uint32MASK(31); BLOK[BLOKlen]._len = 0; BLOK[BLOKlen]._bpf = DATA->tell() / 2; BLOKlen++; // Update the header, assemble the final file. delete DATA; HEAD._magic[0] = SEQSTORE_MAGICNUMBER1; HEAD._magic[1] = SEQSTORE_MAGICNUMBER2; HEAD._pad = uint32ZERO; HEAD._numberOfSequences = nSequences; HEAD._numberOfACGT = nACGT; HEAD._numberOfBlocksACGT = nBlockACGT; HEAD._numberOfBlocksGAP = nBlockGAP; HEAD._numberOfBlocks = BLOKlen; HEAD._namesLength = NAMElen; HEAD._indexStart = uint64ZERO; HEAD._blockStart = uint64ZERO; HEAD._namesStart = uint64ZERO; errno = 0; FILE *F = fopen(filename, "r+"); if (errno) fprintf(stderr, "constructSeqStore()-- Failed to reopen '%s' to write data: %s\n", filename, strerror(errno)), exit(1); fseeko(F, 0, SEEK_END); HEAD._indexStart = ftello(F); fwrite(INDX, sizeof(seqStoreIndex), HEAD._numberOfSequences, F); fseeko(F, 0, SEEK_END); HEAD._blockStart = ftello(F); fwrite(BLOK, sizeof(seqStoreBlock), HEAD._numberOfBlocks, F); fseeko(F, 0, SEEK_END); HEAD._namesStart = ftello(F); fwrite(NAME, sizeof(char), HEAD._namesLength, F); fseeko(F, 0, SEEK_SET); fwrite(&HEAD, sizeof(seqStoreHeader), 1, F); fclose(F); if (errno) fprintf(stderr, "constructSeqStore()-- Failed to write data to '%s': %s\n", filename, strerror(errno)), exit(1); delete [] INDX; delete [] BLOK; delete [] NAME; // ESTmapper depends on this output. fprintf(stderr, "constructSeqStore()-- seqStore '%s' constructed ("F_U32" sequences, "F_U64" ACGT letters, "F_U32" ACGT blocks, "F_U32" GAP blocks).\n", filename, HEAD._numberOfSequences, HEAD._numberOfACGT, HEAD._numberOfBlocksACGT, HEAD._numberOfBlocksGAP); } canu-1.3/src/meryl/libleaff/seqStore.H000066400000000000000000000075731272335422300177230ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef SEQSTORE_H #define SEQSTORE_H #include "seqCache.H" #include "bitPackedFile.H" // A binary fasta file. // // HEADER // magic number // number of sequences // optional - alphabet size // optional - alphabet map (0x00 -> 'a', etc) // position of index start // position of data start // DATA // INDEX // position of sequence start in DATA // header length // sequence length // MAP // name to IID mapping struct seqStoreHeader { uint64 _magic[2]; uint32 _pad; uint32 _numberOfSequences; uint64 _numberOfACGT; uint32 _numberOfBlocksACGT; uint32 _numberOfBlocksGAP; uint32 _numberOfBlocks; uint32 _namesLength; uint64 _indexStart; uint64 _blockStart; uint64 _namesStart; }; // This index allows us to return a complete sequence // struct seqStoreIndex { uint32 _hdrPosition; // Offset into _names for the defline uint32 _hdrLength; // Length of the defline uint64 _seqPosition; // Offset into _bpf for the sequence data uint32 _seqLength; // Length, in bases, of the sequence uint32 _block; // The seqStoreBlock that starts this sequence }; // This index allows us to seek to a specific base in the // file of sequences. Each block is either: // ACGT - and has data // N - no data // It will map a specific ACGT location to the sequence, and the ID // of that sequence (seq ID and location in that sequence). // struct seqStoreBlock { uint64 _isACGT:1; // block is acgt uint64 _pos:32; // position in sequence uint64 _iid:32; // iid of the sequence we are in uint64 _len:23; // length of block uint64 _bpf:40; // position in the bit file of sequence }; #define SEQSTOREBLOCK_MAXPOS uint64MASK(32) #define SEQSTOREBLOCK_MAXIID uint64MASK(32) #define SEQSTOREBLOCK_MAXLEN uint64MASK(23) class seqStore : public seqFile { protected: seqStore(const char *filename); seqStore(); public: ~seqStore(); protected: seqFile *openFile(const char *filename); public: uint32 find(const char *sequencename); uint32 getSequenceLength(uint32 iid); bool getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax); bool getSequence(uint32 iid, uint32 bgn, uint32 end, char *s); private: void clear(void); void loadIndex(void); bitPackedFile *_bpf; seqStoreHeader _header; seqStoreIndex *_index; seqStoreBlock *_block; char *_names; bitPackedFile *_indexBPF; bitPackedFile *_blockBPF; bitPackedFile *_namesBPF; uint32 _lastIIDloaded; friend class seqFactory; }; // Construct a new seqStore 'filename' from input file 'inputseq'. // void constructSeqStore(char *filename, seqCache *inputseq); #endif // SEQSTORE_H canu-1.3/src/meryl/libleaff/seqStream.C000066400000000000000000000241241272335422300200440ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "seqFactory.H" #include "seqStream.H" seqStream::seqStream(const char *filename) { _file = openSeqFile(filename); _string = 0L; _currentIdx = 0; _currentPos = 0; _streamPos = 0; _bufferMax = 1048576; _bufferLen = 0; _bufferPos = 0; _bufferSep = 0; _buffer = new char [_bufferMax + 1]; _idxLen = _file->getNumberOfSequences(); _idx = new seqStreamIndex [_idxLen + 1]; //fprintf(stderr, "seqStream::seqStream()-- Allocating "F_U64"MB for seqStreamIndex on "F_U64" sequences.\n", // _idxLen * sizeof(seqStreamIndex) / 1024 / 1024, _idxLen); _seqNumOfPos = 0L; _lengthOfSequences = 0; _eof = false; _separator = '.'; _separatorLength = 2; setSeparator('.', 2); _bgn = 0; _end = _lengthOfSequences; } seqStream::seqStream(const char *sequence, uint32 length) { _file = 0L; _string = (char *)sequence; _currentIdx = 0; _currentPos = 0; _streamPos = 0; _bufferMax = length; _bufferLen = length; _bufferPos = 0; _bufferSep = 0; _buffer = _string; _idxLen = 1; _idx = new seqStreamIndex [_idxLen + 1]; _seqNumOfPos = 0L; _idx[0]._iid = 0; _idx[0]._len = length; _idx[0]._bgn = 0; _idx[1]._iid = ~uint32ZERO; _idx[1]._len = 0; _idx[1]._bgn = length; _lengthOfSequences = length; _eof = false; _separator = '.'; _separatorLength = 20; _bgn = 0; _end = length; } seqStream::~seqStream() { if (_file) { delete _file; delete [] _buffer; } delete [] _idx; delete [] _seqNumOfPos; } void seqStream::setSeparator(char sep, uint32 len) { // Special case; no separator needed for string backed sequences. if (_string) return; // Bizarre signedness issue with sep=255 // ST->get() == sep FAILS // x=ST->get(); x == sep SUCCEEDS // // Not suggested to use non-printable ascii. if ((isprint(sep) == 0) || (tolower(sep) == 'a') || (tolower(sep) == 'c') || (tolower(sep) == 'g') || (tolower(sep) == 't')) { fprintf(stderr, "seqStream::setSeparator()-- ERROR! Separator letter must be printable ASCII and not [ACGTacgt].\n"); exit(1); } if (len == 0) { fprintf(stderr, "seqStream::setSeparator()-- ERROR! Separator length cannot be zero.\n"); exit(1); } _lengthOfSequences = 0; _separator = sep; _separatorLength = len;; for (uint32 s=0; s<_idxLen; s++) { _idx[s]._iid = s; _idx[s]._len = _file->getSequenceLength(s); _idx[s]._bgn = _lengthOfSequences; _lengthOfSequences += _idx[s]._len; } _idx[_idxLen]._iid = ~uint32ZERO; _idx[_idxLen]._len = 0; _idx[_idxLen]._bgn = _lengthOfSequences; // Rebuild our sequence number of position map, if it exists. // if (_seqNumOfPos) { delete [] _seqNumOfPos; tradeSpaceForTime(); } } void seqStream::tradeSpaceForTime(void) { uint32 i = 0; uint32 s = 0; //fprintf(stderr, "Allocating "F_U32" uint32s for seqNumOfPos.\n", _lengthOfSequences); _seqNumOfPos = new uint32 [_lengthOfSequences]; for (i=0; i<_lengthOfSequences; i++) { // Increment the sequence number until we enter into the next // sequence. Zero length sequences require the use of a 'while' // here. // while (i >= _idx[s+1]._bgn) s++; _seqNumOfPos[i] = s; } } unsigned char seqStream::get(void) { if (_streamPos >= _end) _eof = true; if ((_eof == false) && (_bufferPos >= _bufferLen)) fillBuffer(); if (_eof) return(0); if (_bufferSep == 0) { _currentPos++; _streamPos++; } else { _bufferSep--; } return(_buffer[_bufferPos++]); } void seqStream::rewind(void){ // Search for the correct spot. Uncommon operation, be inefficient // but simple. The range was checked to be good by setRange(). uint32 s = 0; uint64 l = 0; while ((s < _idxLen) && (l + _idx[s]._len < _bgn)) l += _idx[s++]._len; _eof = false; // (_bgn - l) is a 32-bit quanitity because of the second half of // the while above. Although _bgn is a 64-bit value, the value // used to set _bufferPos will be for that of a string constructor, // and so _bgn will be 32-bits. fillBuffer() resets _bufferPos if // we're backed by a file. _currentIdx = s; _currentPos = _bgn - l; _streamPos = _bgn; _bufferPos = _bgn; //fprintf(stderr, "seqStream::rewind()-- 1 currentIdx="F_U32" currentPos="F_U32" streamPos="F_U32" bufferPos="F_U32"\n", // _currentIdx, _currentPos, _streamPos, _bufferPos); fillBuffer(); //fprintf(stderr, "seqStream::rewind()-- 2 currentIdx="F_U32" currentPos="F_U32" streamPos="F_U32" bufferPos="F_U32"\n", // _currentIdx, _currentPos, _streamPos, _bufferPos); } void seqStream::setRange(uint64 bgn, uint64 end) { assert(bgn < end); uint32 s = 0; uint64 l = 0; while (s < _idxLen) l += _idx[s++]._len; if (end == ~uint64ZERO) end = l; if ((bgn > l) || (end > l)) fprintf(stderr, "seqStream::setRange()-- ERROR: range ("F_U64","F_U64") too big; only "F_U64" positions.\n", bgn, end, l), exit(1); _bgn = bgn; _end = end; rewind(); } void seqStream::setPosition(uint64 pos) { assert(_bgn <= pos); assert( pos < _end); uint64 old = _bgn; _bgn = pos; rewind(); _bgn = old; } uint32 seqStream::sequenceNumberOfPosition(uint64 p) { uint32 s = ~uint32ZERO; // binary search on our list of start positions, to find the // sequence that p is in. if (_lengthOfSequences <= p) { fprintf(stderr, "seqStream::sequenceNumberOfPosition()-- WARNING: position p="F_U64" too big; only "F_U64" positions.\n", p, _lengthOfSequences); return(s); } if (_seqNumOfPos) return(_seqNumOfPos[p]); if (_idxLen < 16) { for (s=0; s<_idxLen; s++) if ((_idx[s]._bgn <= p) && (p < _idx[s+1]._bgn)) break; } else { uint32 lo = 0; uint32 hi = _idxLen; uint32 md = 0; while (lo <= hi) { md = (lo + hi) / 2; if (p < _idx[md]._bgn) { // This block starts after the one we're looking for. hi = md; } else if ((_idx[md]._bgn <= p) && (p < _idx[md+1]._bgn)) { // Got it! lo = md + 1; hi = md; s = md; } else { // By default, then, the block is too low. lo = md; } } } return(s); } void seqStream::fillBuffer(void) { // Special case for when we're backed by a character string; there // is no need to fill the buffer. // if (_file == 0L) { if (_currentPos >= _end) _eof = true; return; } // Read bytes from the _file, stuff them into the buffer. Assumes // there is nothing in the buffer to save. _bufferLen = 0; _bufferPos = 0; // Still more stuff in the sequence? Get it. if (_currentPos < _idx[_currentIdx]._len) { #ifdef DEBUG fprintf(stderr, "seqStream::fillBuffer()-- More Seq currentPos="F_U32" len="F_U32"\n", _currentPos, _idx[_currentIdx]._len); #endif _bufferLen = MIN(_idx[_currentIdx]._len - _currentPos, _bufferMax); if (_file->getSequence(_idx[_currentIdx]._iid, _currentPos, _currentPos + _bufferLen, _buffer) == false) fprintf(stderr, "seqStream::fillBuffer()-- Failed to getSequence(part) #1 iid="F_U32" bgn="F_U32" end="F_U32"\n", _idx[_currentIdx]._iid, _currentPos, _currentPos + _bufferLen), exit(1); return; } // We've finished a sequence. Load the next. _currentPos = 0; _currentIdx++; while ((_currentIdx < _idxLen) && (_idx[_currentIdx]._len == 0)) _currentIdx++; #ifdef DEBUG fprintf(stderr, "seqStream::fillBuffer()-- New Seq currentPos="F_U32" len="F_U32"\n", _currentPos, _idx[_currentIdx]._len); #endif // All done if there is no more sequence. if (_currentIdx >= _idxLen) { _eof = true; return; } // Insert a separator. for (_bufferLen = 0; _bufferLen < _separatorLength; _bufferLen++) _buffer[_bufferLen] = _separator; // Keep track of the separator - this is used to make sure we don't // advance the sequence/stream position while the separator is // being returned. // _bufferSep = _bufferLen; // How much to get; minimum of what is left in the sequence, and // the buffer size. Don't forget about the separator we already // inserted! // uint32 bl = MIN(_idx[_currentIdx]._len - _currentPos, _bufferMax - _bufferLen); if (_file->getSequence(_idx[_currentIdx]._iid, _currentPos, _currentPos + bl, _buffer + _bufferLen) == false) fprintf(stderr, "seqStream::fillBuffer()-- Failed to getSequence(part) #2 iid="F_U32" bgn="F_U32" end="F_U32"\n", _idx[_currentIdx]._iid, _currentPos, _currentPos + bl), exit(1); _bufferLen += bl; // Load more, until buffer is full. Not really needed, and won't // improve performance much. AND it adds a lot of complexity to // track which sequence is current (_currentIdx). return; } canu-1.3/src/meryl/libleaff/seqStream.H000066400000000000000000000107371272335422300200560ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef SEQSTREAM_H #define SEQSTREAM_H #include "seqFile.H" struct seqStreamIndex { uint32 _iid; // seqFile IID uint32 _len; // length of the sequence uint64 _bgn; // begin position in the stream }; class seqStream { public: seqStream(const char *filename); seqStream(const char *sequence, uint32 length); ~seqStream(); // Separate sequences with this letter. Non-ACGT is always // returned as 'N'. Changing the length of the separator AFTER // setting the range will result in the wrong range being used. // void setSeparator(char sep, uint32 len); // get() returns one letter per input letter -- a gap of size n // will return n gap symbols. // unsigned char get(void); bool eof(void) { return(_eof); }; // Returns to the start of the range. // void rewind(void); // Set the range of ACGT sequence we will return. Coordinates are // space-based. Example: // // >one // AAA // >two // C // >three // GGG // // We separate these sequences with three '-' letters. // // strPos 012...3...456 // AAA---C---GGG // // range(0,0) -> nothing // range(0,1) -> A // range(0,3) -> AAA // range(0,4) -> AAAnnnC // range(0,5) -> AAAnnnCnnnG // void setRange(uint64 bgn, uint64 end); void setPosition(uint64 pos); // seqPos() is the position we are at in the current sequence; // seqIID() is the iid of that sequence; // strPos() is the position we are at in the chained sequence // // Values are not defined if the letter is a separator. // uint32 seqPos(void) { return(_currentPos); }; uint32 seqIID(void) { return(_idx[_currentIdx]._iid); }; uint64 strPos(void) { return(_streamPos); }; uint32 numberOfSequences(void) { return(_idxLen); }; // Return the length of, position of (in the chain) and IID of the // (s)th sequence in the chain. // uint32 lengthOf(uint32 s) { return((s >= _idxLen) ? ~uint32ZERO : _idx[s]._len); }; uint32 IIDOf(uint32 s) { return((s >= _idxLen) ? ~uint32ZERO : _idx[s]._iid); }; uint64 startOf(uint32 s) { return((s >= _idxLen) ? ~uint64ZERO : _idx[s]._bgn); }; // For a chain position p, returns the s (above) for that position. // uint32 sequenceNumberOfPosition(uint64 p); void tradeSpaceForTime(void); private: void fillBuffer(void); seqFile *_file; // Backed by a seqFile. char *_string; // Backed by a character string. uint64 _bgn; // Begin/End position in chained sequence uint64 _end; uint32 _currentIdx; // index into _idx of the current sequence uint32 _currentPos; // position in the current sequence uint64 _streamPos; // position in the chained sequence // Buffer for holding sequence from the seqFile. uint32 _bufferMax; // Max size of the buffer uint32 _bufferLen; // Actual size of the buffer uint32 _bufferPos; // Position we are at in the buffer uint32 _bufferSep; // How much of the buffer is separator char *_buffer; // Info about the raw sequences uint32 _idxLen; seqStreamIndex *_idx; uint32 *_seqNumOfPos; uint64 _lengthOfSequences; bool _eof; char _separator; uint32 _separatorLength; }; #endif // SEQSTREAM_H canu-1.3/src/meryl/libleaff/sffFile.C000066400000000000000000000141061272335422300174550ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "sffFile.H" #include "bitOperations.H" // Lots of ths came from AS_GKP_sff.c sffFile::sffFile() { clear(); } sffFile::sffFile(const char *name) { clear(); strcpy(_filename, name); _rb = new readBuffer(name); _rb->read(&_header, 31); if (_header.magic_number != 0x2e736666) { _header.swap_endianess = 1; _header.magic_number = uint32Swap(_header.magic_number); _header.index_offset = uint64Swap(_header.index_offset); _header.index_length = uint32Swap(_header.index_length); _header.number_of_reads = uint32Swap(_header.number_of_reads); _header.header_length = uint16Swap(_header.header_length); _header.key_length = uint16Swap(_header.key_length); _header.number_of_flows_per_read = uint16Swap(_header.number_of_flows_per_read); } assert(_header.magic_number == 0x2e736666); assert(_header.number_of_flows_per_read < SFF_NUMBER_OF_FLOWS_MAX); assert(_header.key_length < SFF_KEY_SEQUENCE_MAX); _rb->read(_header.flow_chars, sizeof(char) * _header.number_of_flows_per_read); _rb->read(_header.key_sequence, sizeof(char) * _header.key_length); _firstReadLocation = _header.header_length; // The spec says the index might be here, however, all files I've // seen have the index at the end of the file. // if ((_header.index_length > 0) && (_header.index_offset == _header.header_length)) _firstReadLocation += _header.index_length; // Index // _index = new sffIndex [_header.number_of_reads]; for (uint64 i=0; i<_header.number_of_reads; i++) { uint64 pos = _rb->tell(); _rb->read(&_read, 16); if (_header.swap_endianess) { _read.read_header_length = uint16Swap(_read.read_header_length); _read.name_length = uint16Swap(_read.name_length); _read.number_of_bases = uint32Swap(_read.number_of_bases); } _index[i]._seqPos = pos; _index[i]._seqLen = _read.number_of_bases; _index[i]._namLen = _read.name_length; pos += _read.read_header_length; pos += sizeof(uint16) * _header.number_of_flows_per_read; pos += sizeof(uint8) * _read.number_of_bases; pos += sizeof(char) * _read.number_of_bases; pos += sizeof(uint8) * _read.number_of_bases; pos += (_header.number_of_flows_per_read * sizeof(uint16) + _read.number_of_bases * sizeof(uint8) + _read.number_of_bases * sizeof(char) + _read.number_of_bases * sizeof(uint8)) % 8; _rb->seek(pos); } // // Index _rb->seek(_firstReadLocation); _numberOfSequences = _header.number_of_reads; } sffFile::~sffFile() { delete _rb; delete [] _index; } //////////////////////////////////////// seqFile * sffFile::openFile(const char *name) { struct stat st; // Open the file, return if it matches the SFF magic_number. errno = 0; stat(name, &st); if (errno) return(0L); if ((st.st_mode & S_IFREG) == 0) return(0L); FILE *F = fopen(name, "r"); if (errno) { fprintf(stderr, "sffFile::openFile()- failed to open '%s': %s\n", name, strerror(errno)); return(0L); } uint32 magic_number = 0; AS_UTL_safeRead(F, &magic_number, "sff magic_number", sizeof(uint32), 1); fclose(F); if ((magic_number == 0x2e736666) || (uint32Swap(magic_number) == 0x2e736666)) return(new sffFile(name)); return(0L); } bool sffFile::getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax) { if (iid > _header.number_of_reads) return(false); memset(&_read, 0, sizeof(sffRead)); _rb->seek(_index[iid]._seqPos); _rb->read(&_read, 16); if (_header.swap_endianess) { _read.read_header_length = uint16Swap(_read.read_header_length); _read.name_length = uint16Swap(_read.name_length); _read.number_of_bases = uint32Swap(_read.number_of_bases); _read.clip_quality_left = uint16Swap(_read.clip_quality_left); _read.clip_quality_right = uint16Swap(_read.clip_quality_right); _read.clip_adapter_left = uint16Swap(_read.clip_adapter_left); _read.clip_adapter_right = uint16Swap(_read.clip_adapter_right); } assert(_read.read_header_length < SFF_NAME_LENGTH_MAX); assert(_read.number_of_bases < SFF_NUMBER_OF_BASES_MAX); _rb->read(_read.name, sizeof(char) * _read.name_length); _read.name[_read.name_length] = 0; uint64 pos = _rb->tell(); pos += _read.read_header_length; pos += sizeof(uint16) * _header.number_of_flows_per_read; pos += sizeof(uint8) * _read.number_of_bases; _rb->seek(pos); _rb->read(_read.bases, sizeof(char) * _read.number_of_bases); _read.bases[_read.number_of_bases] = 0; return(true); } bool sffFile::getSequence(uint32 iid, uint32 bgn, uint32 end, char *s) { if (iid > _header.number_of_reads) return(false); // Same as above, mostly. return(false); } void sffFile::clear(void) { memset(_filename, 0, FILENAME_MAX); memset(_typename, 0, FILENAME_MAX); strcpy(_typename, "SFF"); _numberOfSequences = 0; _rb = 0L; memset(&_header, 0, sizeof(sffHeader)); memset(&_read, 0, sizeof(sffRead)); _index = 0L; _firstReadLocation = 0; _readIID = 0; } canu-1.3/src/meryl/libleaff/sffFile.H000066400000000000000000000066051272335422300174670ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef SFF_H #define SFF_H #include "seqFile.H" #define SFF_KEY_SEQUENCE_MAX 64 #define SFF_NAME_LENGTH_MAX 256 #define SFF_NUMBER_OF_FLOWS_MAX 512 #define SFF_NUMBER_OF_BASES_MAX 2048 // The assembler itself cannot handle longer struct sffHeader { // The next block is read in one swoop from the sff file. DO NOT MODIFY! uint32 magic_number; char version[4]; uint64 index_offset; uint32 index_length; uint32 number_of_reads; uint16 header_length; uint16 key_length; uint16 number_of_flows_per_read; uint8 flowgram_format_code; char flow_chars[SFF_NUMBER_OF_FLOWS_MAX]; // h->number_of_flows_per_read char key_sequence[SFF_KEY_SEQUENCE_MAX]; // h->key_length uint32 swap_endianess; }; struct sffRead { // The next block is read in one swoop from the sff file. DO NOT MODIFY! uint16 read_header_length; uint16 name_length; uint32 number_of_bases; uint16 clip_quality_left; uint16 clip_quality_right; uint16 clip_adapter_left; uint16 clip_adapter_right; char name[SFF_NAME_LENGTH_MAX]; // r->name_length uint16 flowgram_values[SFF_NUMBER_OF_FLOWS_MAX]; // h->number_of_flows_per_read uint8 flow_index_per_base[SFF_NUMBER_OF_BASES_MAX]; // r->number_of_bases char bases[SFF_NUMBER_OF_BASES_MAX]; // r->number_of_bases uint8 quality_scores[SFF_NUMBER_OF_BASES_MAX]; // r->number_of_bases char quality[SFF_NUMBER_OF_BASES_MAX]; // quality_scores converted to CA-format qv }; struct sffIndex { uint64 _seqPos; uint32 _seqLen; uint32 _namLen; }; class sffFile : public seqFile { protected: sffFile(const char *filename); sffFile(); public: ~sffFile(); protected: seqFile *openFile(const char *name); public: uint32 find(const char *sequencename) { assert(0); return(0); }; uint32 getSequenceLength(uint32 iid) { return(_index[iid]._seqLen); }; bool getSequence(uint32 iid, char *&h, uint32 &hLen, uint32 &hMax, char *&s, uint32 &sLen, uint32 &sMax); bool getSequence(uint32 iid, uint32 bgn, uint32 end, char *s); private: void clear(void); readBuffer *_rb; sffHeader _header; sffRead _read; sffIndex *_index; uint64 _firstReadLocation; uint64 _readIID; friend class seqFactory; }; #endif // SFF_H canu-1.3/src/meryl/libleaff/test-correctSequence.H000066400000000000000000000105451272335422300222160ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef TEST_CORRECTSEQUENCE_H #define TEST_CORRECTSEQUENCE_H //#define WITH_WHITESPACE struct correctSequence_t { char header[256]; uint32 headerLength; char *sequence; uint32 sequenceLength; }; correctSequence_t *correctSequence = 0L; mt_s *mtctx = 0L; char *chainSeq; uint32 *chainSeqPos; uint32 *chainSeqIID; uint64 *chainStrPos; void generateCorrectSequence(uint32 minLen, uint32 maxLen, uint32 numSeq) { char bases[4] = {'A', 'C', 'G', 'T'}; uint32 n = numSeq; uint32 s = minLen; uint32 l = maxLen; uint32 seed = (uint32)(getTime() * 1000); fprintf(stderr, "generateCorrectSequence()-- Using seed "uint32FMT"\n", seed); fprintf(stderr, "generateCorrectSequence()-- Generating "uint32FMT" sequences of length "uint32FMT" to "uint32FMT"\n", numSeq, minLen, maxLen); correctSequence = new correctSequence_t [n]; mtctx = mtInit(seed); FILE *F = fopen("test-correctSequence.fasta", "w"); for (uint32 i=0; i%s\n", correctSequence[i].header); for (uint32 r=mtRandom32(mtctx) % 4; r--; ) fprintf(F, "\n"); for (uint32 p=0; p%s\n", correctSequence[i].header); fprintf(F, "%s\n", correctSequence[i].sequence); #endif } for (uint32 r=mtRandom32(mtctx) % 4; r--; ) fprintf(F, "\n"); fclose(F); } void generateChainedAnswer(uint32 numSeq, char sep, uint32 sepLen) { uint32 maxLen = 0; for (uint32 i=0; inextMer(); MS->theFMer().merToString(testmer); if (verbose) { fprintf(stdout, "MS pos="uint32FMT" posInSeq="uint64FMT" posInStr="uint64FMT" seqNum="uint64FMT"\n", pos, MS->thePositionInSequence(), MS->thePositionInStream(), MS->theSequenceNumber()); if (strncmp(testmer, seq + pos, merSize)) fprintf(stdout, "MS pos="uint32FMT" failed '%s' != '%s'.\n", pos, testmer, seq + pos); } assert(nm == true); assert(MS->thePositionInSequence() == SP[pos]); assert(MS->thePositionInStream() == SP[pos]); assert(MS->theSequenceNumber() == 0); assert(strncmp(testmer, seq + pos, merSize) == 0); pos++; } // Should have no more mers nm = MS->nextMer(); assert(nm == false); return(err); } uint32 testMerStreamOperation(merStream *MS, uint32 beg, uint32 end, uint32 sepLen) { uint32 err = 0; char fmerstr[256]; char rmerstr[256]; char cmerstr[256]; char tmerstr[256]; while (MS->nextMer()) { MS->theFMer().merToString(fmerstr); MS->theRMer().merToString(rmerstr); MS->theCMer().merToString(cmerstr); if ((strcmp(fmerstr, cmerstr) != 0) && ((strcmp(rmerstr, cmerstr) != 0))) { fprintf(stderr, "mer strings disagree; F:%s R:%s C:%s\n", fmerstr, rmerstr, cmerstr); FAIL(); } reverseComplementSequence(rmerstr, strlen(rmerstr)); if (strcmp(fmerstr, rmerstr) != 0) { fprintf(stderr, "mer strings disagree after reverse; F:%s R:%s\n", fmerstr, rmerstr); FAIL(); } uint32 pseq = MS->thePositionInSequence(); uint32 pstr = MS->thePositionInStream(); uint32 piid = MS->theSequenceNumber(); uint32 mersize = MS->theFMer().getMerSize(); uint32 merspan = MS->theFMer().getMerSpan(); #if 0 if (beg > 10) { uint32 pp = pstr + piid * sepLen - 10; uint32 xx = 0; fprintf(stderr, "beg="uint32FMT" pstr="uint32FMT" '", beg, pstr); for (xx=0; xx<10; xx++, pp++) fprintf(stderr, "%c", chainSeq[pp]); fprintf(stderr, ":"); for (xx=0; xx 1) { ST = new seqStream("test-correctSequence.fasta"); ST->setSeparator(sep, sepLen); } else { ST = new seqStream(correctSequence[0].sequence, correctSequence[0].sequenceLength); } MS = new merStream(KB, ST, true, true); uint32 maxLen = ST->startOf(numSeq-1) + ST->lengthOf(numSeq-1); // Whole thing, rewind, whole thing fprintf(stderr, "whole thing.\n"); err += testMerStreamOperation(MS, 0, maxLen, sepLen); MS->rewind(); err += testMerStreamOperation(MS, 0, maxLen, sepLen); // Random subsets - we're not terribly interested in streaming, // just getting the start/end correct. fprintf(stderr, "subsets.\n"); for (uint32 iter=0; iter<500; iter++) { uint32 beg = mtRandom32(mtctx) % maxLen; uint32 end = (beg + 10000 < maxLen) ? (beg + 10000) : maxLen; //fprintf(stderr, "subsets - "uint32FMT"-"uint32FMT"\n", beg, end); MS->setBaseRange(beg, end); err += testMerStreamOperation(MS, beg, end, sepLen); MS->rewind(); err += testMerStreamOperation(MS, beg, end, sepLen); } delete MS; return(err); } int main(int argc, char **argv) { uint32 minLen = 1000; uint32 maxLen = 200000; uint32 numSeq = 1000; uint32 err = 0; // Very simple merStream test { fprintf(stdout, "merStream(kMerBuilder(20), ...)\n"); merStream *MS = new merStream(new kMerBuilder(20), new seqStream("GGGTCAACTCCGCCCGCACTCTAGC", 25), true, true); uint32 SP[10] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; testMerStreamSimple(MS, 20, "GGGTCAACTCCGCCCGCACTCTAGC", SP); MS->rewind(); testMerStreamSimple(MS, 20, "GGGTCAACTCCGCCCGCACTCTAGC", SP); MS->rewind(); MS->rewind(); testMerStreamSimple(MS, 20, "GGGTCAACTCCGCCCGCACTCTAGC", SP); delete MS; fprintf(stdout, "merStream(kMerBuilder(20), ...) - PASSED\n"); } { fprintf(stdout, "merStream(kMerBuilder(20, 1), ...)\n"); merStream *MS = new merStream(new kMerBuilder(20, 1), new seqStream("GGGAATTTTCAACTCCGCCCGCACTCTAGCCCAAA", 35), true, true); uint32 SP[10] = { 0, 3, 5, 9, 10, 12 }; testMerStreamSimple(MS, 20, "GATCACTCGCGCACTCTAGCA", SP); MS->rewind(); testMerStreamSimple(MS, 20, "GATCACTCGCGCACTCTAGCA", SP); MS->rewind(); MS->rewind(); testMerStreamSimple(MS, 20, "GATCACTCGCGCACTCTAGCA", SP); delete MS; fprintf(stdout, "merStream(kMerBuilder(20, 1), ...) - PASSED\n"); } // Move on to harder tests generateCorrectSequence(minLen, maxLen, numSeq); // Tests seqStream(string, strlen) construction method fprintf(stderr, "err += testMerStream(new kMerBuilder(20, 0, 0L), 1, '.', 1);\n"); err += testMerStream(new kMerBuilder(20, 0, 0L), 1, '.', 1); fprintf(stderr, "err += testMerStream(new kMerBuilder(22, 1, 0L), 1, '.', 1);\n"); err += testMerStream(new kMerBuilder(22, 1, 0L), 1, '.', 1); // Tests seqStream(filename) construction method fprintf(stderr, "err += testMerStream(new kMerBuilder(20, 0, 0L), numSeq, '.', 1);\n"); err += testMerStream(new kMerBuilder(20, 0, 0L), numSeq, '.', 1); fprintf(stderr, "err += testMerStream(new kMerBuilder(28, 0, 0L), numSeq, '.', 100);\n"); err += testMerStream(new kMerBuilder(28, 0, 0L), numSeq, '.', 100); fprintf(stderr, "err += testMerStream(new kMerBuilder(24, 4, 0L), numSeq, '.', 100);\n"); err += testMerStream(new kMerBuilder(24, 4, 0L), numSeq, '.', 100); removeCorrectSequence(numSeq); if (err == 0) fprintf(stderr, "Success!\n"); exit(err > 0); } canu-1.3/src/meryl/libleaff/test-seqCache.C000066400000000000000000000134731272335422300205760ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "util.h" #include "seqCache.H" #include "seqStream.H" #include "merStream.H" #include "test-correctSequence.H" uint32 testSeqVsCorrect(seqInCore *S, uint32 testID) { uint32 err = 0; if (S == 0L) { fprintf(stderr, "testID:"uint32FMT" - empty sequence\n", testID); return(1); } uint32 sid = S->getIID(); if (strcmp(S->header(), correctSequence[sid].header) != 0) { fprintf(stderr, "testID:"uint32FMT" - header differs '%s' vs '%s'\n", testID, S->header(), correctSequence[sid].header); err++; } if (S->headerLength() != correctSequence[sid].headerLength) { fprintf(stderr, "testID:"uint32FMT" - header length differs "uint32FMT" vs "uint32FMT"\n", testID, S->headerLength(), correctSequence[sid].headerLength); err++; } if (strcmp(S->sequence(), correctSequence[sid].sequence) != 0) { fprintf(stderr, "testID:"uint32FMT" - sequence differs\n", testID); err++; } if (strlen(S->sequence()) != correctSequence[sid].sequenceLength) { fprintf(stderr, "testID:"uint32FMT" - sequence length differs strlen "uint32FMT" vs "uint32FMT"\n", testID, (uint32)strlen(S->sequence()), correctSequence[sid].sequenceLength); err++; } if (S->sequenceLength() != correctSequence[sid].sequenceLength) { fprintf(stderr, "testID:"uint32FMT" - sequence length differs "uint32FMT" vs "uint32FMT"\n", testID, S->sequenceLength(), correctSequence[sid].sequenceLength); err++; } return(err); } uint32 testSeqCacheIDLookups(seqCache *SC) { uint32 err = 0; uint32 numSeq = SC->getNumberOfSequences(); double start = getTime(); // 1 - getSequenceIID() fprintf(stderr, "1 - getSequenceIID()\n"); for (uint32 sid=0; sidgetSequenceIID(correctSequence[sid].header)) { fprintf(stderr, "2 - failed to find name '%s'\n", correctSequence[sid].header); err++; } } fprintf(stderr, "Test took %f seconds.\n", getTime() - start); return(err); } uint32 testSeqCache(seqCache *SC) { uint32 err = 0; uint32 numSeq = SC->getNumberOfSequences(); seqInCore *S = 0L; double start = getTime(); // 0 - getSequenceLength() fprintf(stderr, "0 - getSequenceLength()\n"); for (uint32 sid=0; sidgetSequenceLength(sid) != correctSequence[sid].sequenceLength) { fprintf(stderr, "1 - length differs.\n"); err++; } // 2 - stream with getSequenceInCore() fprintf(stderr, "2 - stream with getSequenceInCore()\n"); S = SC->getSequenceInCore(); while (S != 0L) { err += testSeqVsCorrect(S, 2); delete S; S = SC->getSequenceInCore(); } // 3 - iterate with getSequenceInCore(sid++) fprintf(stderr, "3 - iterate with getSequenceInCore(sid++)\n"); for (uint32 sid=0; sidgetSequenceInCore(sid); err += testSeqVsCorrect(S, 3); delete S; } // 4 - random with getSequenceInCore(sid) fprintf(stderr, "4 - random with getSequenceInCore(sid)\n"); for (uint32 cnt=0; cnt<4*numSeq; cnt++) { uint32 sid = mtRandom32(mtctx) % numSeq; S = SC->getSequenceInCore(sid); err += testSeqVsCorrect(S, 4); delete S; } fprintf(stderr, "Test took %f seconds.\n", getTime() - start); return(err); } int main(int argc, char **argv) { uint32 minLen = 100; uint32 maxLen = 2000; uint32 numSeq = 100000; seqCache *SC = 0L; uint32 err = 0; generateCorrectSequence(minLen, maxLen, numSeq); fprintf(stderr, "seqCache(file, 0, true) (ID lookups)\n"); SC = new seqCache("test-correctSequence.fasta", 0, true); //err += testSeqCacheIDLookups(SC); delete SC; fprintf(stderr, "seqCache(file, 0, true)\n"); SC = new seqCache("test-correctSequence.fasta", 0, true); err += testSeqCache(SC); delete SC; fprintf(stderr, "seqCache(file, 1, true)\n"); SC = new seqCache("test-correctSequence.fasta", 1, true); err += testSeqCache(SC); delete SC; fprintf(stderr, "seqCache(file, 2, true)\n"); SC = new seqCache("test-correctSequence.fasta", 2, true); err += testSeqCache(SC); delete SC; fprintf(stderr, "seqCache(file, 4, true)\n"); SC = new seqCache("test-correctSequence.fasta", 4, true); err += testSeqCache(SC); delete SC; fprintf(stderr, "seqCache(file, 8, true)\n"); SC = new seqCache("test-correctSequence.fasta", 8, true); err += testSeqCache(SC); delete SC; fprintf(stderr, "seqCache(file, 32, true)\n"); SC = new seqCache("test-correctSequence.fasta", 32, true); err += testSeqCache(SC); delete SC; fprintf(stderr, "seqCache(file, 200, true)\n"); SC = new seqCache("test-correctSequence.fasta", 200, true); err += testSeqCache(SC); delete SC; fprintf(stderr, "seqCache(file, 1000000, true)\n"); SC = new seqCache("test-correctSequence.fasta", 1000000, true); err += testSeqCache(SC); delete SC; fprintf(stderr, "seqCache(file, 0, true) -- loadAllSequence\n"); SC = new seqCache("test-correctSequence.fasta", 0, true); SC->loadAllSequences(); err += testSeqCache(SC); delete SC; removeCorrectSequence(numSeq); if (err == 0) fprintf(stderr, "Success!\n"); exit(err > 0); } canu-1.3/src/meryl/libleaff/test-seqStream.C000066400000000000000000000205331272335422300210210ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "util.h" #include "seqCache.H" #include "seqStream.H" #include "merStream.H" #include "test-correctSequence.H" #define FAIL() { err++; assert(0); } uint32 testIndexing(uint32 numSeq, char sep, uint32 sepLen) { uint32 err = 0; seqStream *ST = 0L; fprintf(stderr, "testIndexing()-- numSeq="uint32FMT" sep=%c sepLen="uint32FMT"\n", numSeq, sep, sepLen); generateChainedAnswer(numSeq, sep, sepLen); if (numSeq > 1) { ST = new seqStream("test-correctSequence.fasta"); ST->setSeparator(sep, sepLen); } else { ST = new seqStream(correctSequence[0].sequence, correctSequence[0].sequenceLength); } uint32 maxLen = ST->startOf(numSeq-1) + ST->lengthOf(numSeq-1); // Basic checks on the reverse lookup - this is state independent; // it changes only based on the separator length. In other words, // there is no need to check this while iterating through the // seqStream. fprintf(stderr, "IGNORE THIS WARNING: "); if (ST->sequenceNumberOfPosition(maxLen) != ~uint32ZERO) { fprintf(stderr, "maxLen too small.\n"); FAIL(); } if (ST->sequenceNumberOfPosition(maxLen - 1) == ~uint32ZERO) { fprintf(stderr, "maxLen too big.\n"); FAIL(); } // Check all lookups - lengthOf() and IIDOf() are implicitly // checked by the operation of seqStream (get() mostly). startOf() // isn't, but inserting errors in setRange() led to // infinite-looking loops. uint64 pos = 0; uint64 sta = 0; for (uint32 sid=0; sidlengthOf(sid) != correctSequence[sid].sequenceLength) { fprintf(stderr, "lengthOf "uint32FMT" returned "uint32FMT", not correct "uint32FMT"\n", sid, ST->lengthOf(sid), correctSequence[sid].sequenceLength); FAIL(); } if (ST->startOf(sid) != sta) { fprintf(stderr, "startOf "uint32FMT" returned "uint64FMT", not correct "uint64FMT"\n", sid, ST->startOf(sid), sta); FAIL(); } if (ST->IIDOf(sid) != sid) { fprintf(stderr, "IIDOf "uint32FMT" returned "uint32FMT", not correct "uint32FMT"\n", sid, ST->IIDOf(sid), sid); FAIL(); } sta += correctSequence[sid].sequenceLength; for (uint32 ppp=0; pppsequenceNumberOfPosition(pos) != sid) { fprintf(stderr, "sequenceNumberOfPosition "uint64FMT" returned "uint32FMT", not correct "uint32FMT".\n", pos, ST->sequenceNumberOfPosition(pos), sid); FAIL(); } } } if (pos != maxLen) { fprintf(stderr, "maxLen wrong.\n"); FAIL(); } // Check the separator. Seek to a spot right before one, and count // that we have the correct length. More rigorously tested in // testChaining(). for (uint32 sid=0; sidsetRange(ST->startOf(sid) + ST->lengthOf(sid)-1, ~uint64ZERO); ST->get(); for (uint32 x=0; xget(); if (s != sep) { fprintf(stderr, "wrong separator at sep "uint32FMT" got %d expected %d\n", x, s, sep); FAIL(); } } if (ST->get() == sep) { fprintf(stderr, "too many separators!\n"); FAIL(); } } delete ST; return(err); } uint32 testSeqStream(seqStream *ST, uint32 sib, uint32 sie, char sep) { uint32 err = 0; while (ST->eof() == false) { uint32 sp = ST->seqPos(); uint32 si = ST->seqIID(); uint64 st = ST->strPos(); char ch = ST->get(); if (ch != 0) { if (ch != chainSeq[sib]) { fprintf(stderr, "sp="uint32FMT" si="uint32FMT" st="uint64FMT" ch=%c -- letter wrong got'%c'\n", sp, si, st, ch, chainSeq[sib]); FAIL(); } if ((ch != sep) && (sp != chainSeqPos[sib])) { fprintf(stderr, "sp="uint32FMT" si="uint32FMT" st="uint64FMT" ch=%c -- seqPos wrong got "uint32FMT"\n", sp, si, st, ch, chainSeqPos[sib]); FAIL(); } if ((ch != sep) && (si != chainSeqIID[sib])) { fprintf(stderr, "sp="uint32FMT" si="uint32FMT" st="uint64FMT" ch=%c -- seqIID wrong got"uint32FMT"\n", sp, si, st, ch, chainSeqIID[sib]); FAIL(); } if ((ch != sep) && (st != chainStrPos[sib])) { fprintf(stderr, "sp="uint32FMT" si="uint32FMT" st="uint64FMT" ch=%c -- strPos wrong got "uint64FMT"\n", sp, si, st, ch, chainStrPos[sib]); FAIL(); } sib++; } } if (sib != sie) { fprintf(stderr, "iterated length wrong; sib="uint32FMT" sie="uint32FMT"\n", sib, sie); FAIL(); } return(err); } uint32 testChaining(uint32 numSeq, char sep, uint32 sepLen) { uint32 err = 0; seqStream *ST = 0L; fprintf(stderr, "testChaining()-- numSeq="uint32FMT" sep=%c sepLen="uint32FMT"\n", numSeq, sep, sepLen); generateChainedAnswer(numSeq, sep, sepLen); if (numSeq > 1) { ST = new seqStream("test-correctSequence.fasta"); ST->setSeparator(sep, sepLen); } else { ST = new seqStream(correctSequence[0].sequence, correctSequence[0].sequenceLength); } // Do a test on the whole thing. { uint32 sib = 0; uint32 sie = strlen(chainSeq); fprintf(stderr, "initial test with full range\n"); testSeqStream(ST, sib, sie, sep); fprintf(stderr, "initial test with full range (rewind)\n"); ST->rewind(); testSeqStream(ST, sib, sie, sep); } // Set the range to random values, and check all the results. // We've already verified the index works, so we're free to use // that (but we currently don't). uint32 maxLen = ST->startOf(numSeq-1) + ST->lengthOf(numSeq-1); fprintf(stderr, "test on subranges\n"); for (uint32 iter=0; iter<500; iter++) { uint32 beg = mtRandom32(mtctx) % maxLen; uint32 end = mtRandom32(mtctx) % maxLen; if (beg > end) { uint32 t = end; end = beg; beg = t; } ST->setRange(beg, end); // Compute the position in our stream for the ACGT based beg and // end. The quirk here is that our stream includes the // separator. uint32 sib = 0; // chainSeq position uint32 sie = 0; for (uint32 ppp=0, sid=0; sideof() == false) ST->get(); ST->rewind(); } else { //fprintf(stderr, "Random iter "uint32FMT"\n", iter); } testSeqStream(ST, sib, sie, sep); } return(err > 0); } int main(int argc, char **argv) { uint32 minLen = 100; uint32 maxLen = 20000; uint32 numSeq = 1000; uint32 err = 0; generateCorrectSequence(minLen, maxLen, numSeq); // Tests seqStream(string, strlen) construction method err += testIndexing(1, '.', 1); err += testChaining(1, '.', 1); // Tests seqStream(filename) construction method err += testIndexing(numSeq, '.', 1); err += testIndexing(numSeq, ':', 10); err += testIndexing(numSeq, 'z', 100); err += testIndexing(numSeq, '-', 1000); err += testChaining(numSeq, '.', 1); err += testChaining(numSeq, ':', 10); err += testChaining(numSeq, 'z', 100); err += testChaining(numSeq, '-', 1000); removeCorrectSequence(numSeq); if (err == 0) fprintf(stderr, "Success!\n"); exit(err > 0); } canu-1.3/src/meryl/libleaff/test/000077500000000000000000000000001272335422300167505ustar00rootroot00000000000000canu-1.3/src/meryl/libleaff/test/Makefile000066400000000000000000000011671272335422300204150ustar00rootroot00000000000000 PROG = test-merstream-speed INCLUDE = -I.. -I../../libutil -I../../libbio -I../../libseq LIBS = -L.. -L../../libutil -L../../libbio -L../../libseq -lseq -lbio -lutil -lm OBJS = include ../../Make.compilers all: $(PROG) @echo Tests passed! test-merstream-speed: test-merstream-speed.C $(CXX) $(CXXFLAGS_COMPILE) -c -o test-merstream-speed.o test-merstream-speed.C $(INCLUDE) $(CXX) $(CXXLDFLAGS) -o test-merstream-speed test-merstream-speed.o $(LIBS) ../../leaff/leaff -G 10000 1000 10000 > junk.fasta cat junk.fasta > /dev/null ./test-merstream-speed junk.fasta rm -f junk* clean: rm -f $(PROG) *.o *junk* canu-1.3/src/meryl/libleaff/test/test-merstream-speed.C000066400000000000000000000041531272335422300231310ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include "bio++.H" #include "seqCache.H" #include "seqStream.H" #include "merStream.H" int main(int argc, char **argv) { speedCounter *C = 0L; FILE *F = 0L; seqStream *S = 0L; merStream *M = 0L; if (argc != 2) { fprintf(stderr, "usage: %s some.fasta\n", argv[0]); fprintf(stderr, "Reads some.fasta using fgetc(), the seqStream and the merStream,\n"); fprintf(stderr, "reporting the speed of each method.\n"); exit(1); } //////////////////////////////////////// F = fopen(argv[1], "r"); C = new speedCounter("fgetc(): %7.2f Mthings -- %5.2f Mthings/second\r", 1000000.0, 0x3fffff, true); while (!feof(F)) fgetc(F), C->tick(); delete C; fclose(F); //////////////////////////////////////// S = new seqStream(argv[1]); C = new speedCounter("seqStream: %7.2f Mthings -- %5.2f Mthings/second\r", 1000000.0, 0x3fffff, true); while (S->get()) C->tick(); delete C; delete S; //////////////////////////////////////// M = new merStream(new kMerBuilder(20), new seqStream(argv[1]), true, true); C = new speedCounter("seqStream -> merStream: %7.2f Mthings -- %5.2f Mthings/second\r", 1000000.0, 0x3fffff, true); while (M->nextMer()) C->tick(); delete C; delete M; exit(0); } canu-1.3/src/meryl/libmeryl.C000066400000000000000000000404661272335422300161620ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libmeryl/libmeryl.C * * Modifications by: * * Brian P. Walenz from 2003-SEP-08 to 2004-APR-08 * are Copyright 2003-2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2004-APR-30 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAY-23 to 2014-APR-11 * are Copyright 2005-2008,2012,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2015-JUL-01 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "libmeryl.H" #include "AS_UTL_fileIO.H" #include "AS_UTL_alloc.H" // Version 3 ?? // Version 4 removed _histogramHuge, dynamically sizing it on write. // 0123456789012345 static char *ImagicV = "merylStreamIv04\n"; static char *ImagicX = "merylStreamIvXX\n"; static char *DmagicV = "merylStreamDv04\n"; static char *DmagicX = "merylStreamDvXX\n"; static char *PmagicV = "merylStreamPv04\n"; static char *PmagicX = "merylStreamPvXX\n"; merylStreamReader::merylStreamReader(const char *fn_, uint32 ms_) { if (fn_ == 0L) { fprintf(stderr, "ERROR - no counted database file specified.\n"); exit(1); } memset(_filename, 0, sizeof(char) * FILENAME_MAX); strcpy(_filename, fn_); // Open the files // char *inpath = new char [strlen(_filename) + 8]; sprintf(inpath, "%s.mcidx", _filename); _IDX = new bitPackedFile(inpath); sprintf(inpath, "%s.mcdat", _filename); _DAT = new bitPackedFile(inpath); sprintf(inpath, "%s.mcpos", _filename); if (AS_UTL_fileExists(inpath)) _POS = new bitPackedFile(inpath); else _POS = 0L; delete [] inpath; // Verify that they are what they should be, and read in the header // char Imagic[16] = {0}; char Dmagic[16] = {0}; char Pmagic[16] = {0}; bool fail = false; for (uint32 i=0; i<16; i++) { Imagic[i] = _IDX->getBits(8); Dmagic[i] = _DAT->getBits(8); if (_POS) Pmagic[i] = _POS->getBits(8); } if (strncmp(Imagic, ImagicX, 16) == 0) { fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcidx is an INCOMPLETE merylStream index file!\n", _filename); fail = true; } if (strncmp(Imagic, ImagicX, 13) != 0) { fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcidx is not a merylStream index file!\n", _filename); fail = true; } if (strncmp(Dmagic, DmagicX, 16) == 0) { fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcdat is an INCOMPLETE merylStream data file!\n", _filename); fail = true; } if (strncmp(Dmagic, DmagicX, 13) != 0) { fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcdat is not a merylStream data file!\n", _filename); fail = true; } if ((Imagic[13] != Dmagic[13]) || (Imagic[14] != Dmagic[14])) { fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcidx and %s.mcdat are different versions!\n", _filename, _filename); fail = true; } if (_POS) { if (strncmp(Pmagic, PmagicX, 16) == 0) { fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcpos is an INCOMPLETE merylStream data file!\n", _filename); fail = true; } if (strncmp(Pmagic, PmagicX, 13) != 0) { fprintf(stderr, "merylStreamReader()-- ERROR: %s.mcpos is not a merylStream data file!\n", _filename); fail = true; } } if (fail) exit(1); _idxIsPacked = _IDX->getBits(32); _datIsPacked = _IDX->getBits(32); _posIsPacked = _IDX->getBits(32); _merSizeInBits = _IDX->getBits(32) << 1; _merCompression = _IDX->getBits(32); _prefixSize = _IDX->getBits(32); _merDataSize = _merSizeInBits - _prefixSize; _numUnique = _IDX->getBits(64); _numDistinct = _IDX->getBits(64); _numTotal = _IDX->getBits(64); _histogramPos = 0; _histogramLen = 0; _histogramMaxValue = 0; _histogram = 0L; uint32 version = atoi(Imagic + 13); // Versions earlier than four used a fixed-size histogram, stored at the start // of the index. if (version < 4) { _histogramPos = _IDX->tell(); _histogramLen = _IDX->getBits(64); // Previous _histogramHuge, now unused _histogramLen = _IDX->getBits(64); _histogramMaxValue = _IDX->getBits(64); _histogram = new uint64 [_histogramLen]; for (uint32 i=0; i<_histogramLen; i++) _histogram[i] = _IDX->getBits(64); } // Version 4 switched to a dynamically sized histogram, stored at the end // of the index. else { _histogramPos = _IDX->getBits(64); _histogramLen = _IDX->getBits(64); _histogramMaxValue = _IDX->getBits(64); _histogram = new uint64 [_histogramLen]; uint64 position = _IDX->tell(); _IDX->seek(_histogramPos); for (uint32 i=0; i<_histogramLen; i++) _histogram[i] = _IDX->getBits(64); _IDX->seek(position); } _thisBucket = uint64ZERO; _thisBucketSize = getIDXnumber(); _numBuckets = uint64ONE << _prefixSize; _thisMer.setMerSize(_merSizeInBits >> 1); _thisMer.clear(); _thisMerCount = uint64ZERO; _thisMerPositionsMax = 0; _thisMerPositions = 0L; _validMer = true; #ifdef SHOW_VARIABLES fprintf(stderr, "_merSizeInBits = "F_U32"\n", _merSizeInBits); fprintf(stderr, "_merCompression = "F_U32"\n", _merCompression); fprintf(stderr, "_prefixSize = "F_U32"\n", _prefixSize); fprintf(stderr, "_merDataSize = "F_U32"\n", _merDataSize); fprintf(stderr, "_numUnique = "F_U64"\n", _numUnique); fprintf(stderr, "_numDistinct = "F_U64"\n", _numDistinct); fprintf(stderr, "_numTotal = "F_U64"\n", _numTotal); fprintf(stderr, "_thisBucket = "F_U64"\n", _thisBucket); fprintf(stderr, "_thisBucketSize = "F_U64"\n", _thisBucketSize); fprintf(stderr, "_thisMerCount = "F_U64"\n", _thisMerCount); #endif if ((ms_ > 0) && (_merSizeInBits >> 1 != ms_)) { fprintf(stderr, "merylStreamReader()-- ERROR: User requested mersize "F_U32" but '%s' is mersize "F_U32"\n", ms_, _filename, _merSizeInBits >> 1); exit(1); } } merylStreamReader::~merylStreamReader() { delete _IDX; delete _DAT; delete _POS; delete [] _thisMerPositions; delete [] _histogram; } bool merylStreamReader::nextMer(void) { // Use a while here, so that we skip buckets that are empty // while ((_thisBucketSize == 0) && (_thisBucket < _numBuckets)) { _thisBucketSize = getIDXnumber(); _thisBucket++; } if (_thisBucket >= _numBuckets) return(_validMer = false); // Before you get rid of the clear() -- if, say, the list of mers // is sorted and we can shift the mer to make space for the new // stuff -- make sure that nobody is calling reverseComplement()! // _thisMer.clear(); _thisMer.readFromBitPackedFile(_DAT, _merDataSize); _thisMer.setBits(_merDataSize, _prefixSize, _thisBucket); _thisMerCount = getDATnumber(); _thisBucketSize--; if (_POS) { if (_thisMerPositionsMax < _thisMerCount) { delete [] _thisMerPositions; _thisMerPositionsMax = _thisMerCount + 1024; _thisMerPositions = new uint32 [_thisMerPositionsMax]; } for (uint32 i=0; i<_thisMerCount; i++) { _thisMerPositions[i] = _POS->getBits(32); } } return(true); } merylStreamWriter::merylStreamWriter(const char *fn_, uint32 merSize, uint32 merComp, uint32 prefixSize, bool positionsEnabled) { memset(_filename, 0, sizeof(char) * FILENAME_MAX); strcpy(_filename, fn_); char *outpath = new char [FILENAME_MAX]; sprintf(outpath, "%s.mcidx.creating", _filename); _IDX = new bitPackedFile(outpath, 0, true); sprintf(outpath, "%s.mcdat.creating", _filename); _DAT = new bitPackedFile(outpath, 0, true); if (positionsEnabled) { sprintf(outpath, "%s.mcpos.creating", _filename); _POS = new bitPackedFile(outpath, 0, true); } else { _POS = 0L; } delete [] outpath; _idxIsPacked = 1; _datIsPacked = 1; _posIsPacked = 0; _merSizeInBits = merSize * 2; _merCompression = merComp; _prefixSize = prefixSize; _merDataSize = _merSizeInBits - _prefixSize; _thisBucket = uint64ZERO; _thisBucketSize = uint64ZERO; _numBuckets = uint64ONE << _prefixSize; _numUnique = uint64ZERO; _numDistinct = uint64ZERO; _numTotal = uint64ZERO; _histogramPos = 0; _histogramLen = 1024; _histogramMaxValue = 0; _histogram = new uint64 [_histogramLen]; for (uint32 i=0; i<_histogramLen; i++) _histogram[i] = 0; _thisMerIsBits = false; _thisMerIskMer = false; _thisMer.setMerSize(_merSizeInBits >> 1); _thisMer.clear(); _thisMerPre = uint64ZERO; _thisMerMer = uint64ZERO; _thisMerPreSize = prefixSize; _thisMerMerSize = 2 * merSize - prefixSize; _thisMerCount = uint64ZERO; // Initialize the index file. for (uint32 i=0; i<16; i++) _IDX->putBits(ImagicX[i], 8); _IDX->putBits(_idxIsPacked, 32); _IDX->putBits(_datIsPacked, 32); _IDX->putBits(_posIsPacked, 32); _IDX->putBits(_merSizeInBits >> 1, 32); _IDX->putBits(_merCompression, 32); _IDX->putBits(_prefixSize, 32); _IDX->putBits(_numUnique, 64); _IDX->putBits(_numDistinct, 64); _IDX->putBits(_numTotal, 64); _IDX->putBits(0, 64); // Offset to the histogram _IDX->putBits(0, 64); // Length of the histogram data _IDX->putBits(0, 64); // Max value seen in the histogram // Initialize the data file. for (uint32 i=0; i<16; i++) _DAT->putBits(DmagicX[i], 8); // Initialize the positions file. if (_POS) for (uint32 i=0; i<16; i++) _POS->putBits(PmagicX[i], 8); } merylStreamWriter::~merylStreamWriter() { writeMer(); // Finish writing the buckets. while (_thisBucket < _numBuckets + 2) { setIDXnumber(_thisBucketSize); _thisBucketSize = 0; _thisBucket++; } // Save the position of the histogram _histogramPos = _IDX->tell(); // And write the histogram for (uint32 i=0; i<=_histogramMaxValue; i++) _IDX->putBits(_histogram[i], 64); // Seek back to the start and rewrite the magic numbers. _IDX->seek(0); for (uint32 i=0; i<16; i++) _IDX->putBits(ImagicV[i], 8); _IDX->putBits(_idxIsPacked, 32); _IDX->putBits(_datIsPacked, 32); _IDX->putBits(_posIsPacked, 32); _IDX->putBits(_merSizeInBits >> 1, 32); _IDX->putBits(_merCompression, 32); _IDX->putBits(_prefixSize, 32); _IDX->putBits(_numUnique, 64); _IDX->putBits(_numDistinct, 64); _IDX->putBits(_numTotal, 64); _IDX->putBits(_histogramPos, 64); _IDX->putBits(_histogramMaxValue+1, 64); // The length of the data (includes 0) _IDX->putBits(_histogramMaxValue, 64); // The maximum value of the data delete _IDX; delete [] _histogram; // Seek back to the start of the data and rewrite the magic numbers. _DAT->seek(0); for (uint32 i=0; i<16; i++) _DAT->putBits(DmagicV[i], 8); delete _DAT; // Seek back to the start of the positions and rewrite the magic numbers. if (_POS) { _POS->seek(0); for (uint32 i=0; i<16; i++) _POS->putBits(PmagicV[i], 8); } delete _POS; // All done! Rename our temporary outputs to final outputs. char *outpath = new char [FILENAME_MAX]; char *finpath = new char [FILENAME_MAX]; sprintf(outpath, "%s.mcidx.creating", _filename); sprintf(finpath, "%s.mcidx", _filename); rename(outpath, finpath); sprintf(outpath, "%s.mcdat.creating", _filename); sprintf(finpath, "%s.mcdat", _filename); rename(outpath, finpath); if (_POS) { sprintf(outpath, "%s.mcpos.creating", _filename); sprintf(finpath, "%s.mcpos", _filename); rename(outpath, finpath); } delete [] finpath; delete [] outpath; } void merylStreamWriter::writeMer(void) { if (_thisMerCount == 0) return; _numTotal += _thisMerCount; _numDistinct++; if (_thisMerCount >= _histogramLen) resizeArray(_histogram, _histogramMaxValue, _histogramLen, _thisMerCount + 16384); _histogram[_thisMerCount]++; if (_histogramMaxValue < _thisMerCount) _histogramMaxValue = _thisMerCount; assert((_thisMerIsBits == false) || (_thisMerIskMer == false)); if (_thisMerIsBits) { if (_thisMerCount == 1) { _DAT->putBits(_thisMerMer, _thisMerMerSize); setDATnumber(1); _thisBucketSize++; _numUnique++; } else { _DAT->putBits(_thisMerMer, _thisMerMerSize); setDATnumber(_thisMerCount); _thisBucketSize++; } } else { if (_thisMerCount == 1) { _thisMer.writeToBitPackedFile(_DAT, _merDataSize); setDATnumber(1); _thisBucketSize++; _numUnique++; } else if (_thisMerCount > 1) { _thisMer.writeToBitPackedFile(_DAT, _merDataSize); setDATnumber(_thisMerCount); _thisBucketSize++; } } } void merylStreamWriter::addMer(kMer &mer, uint32 count, uint32 *positions) { uint64 val; if (_thisMerIskMer == false) { _thisMerIskMer = true; assert(_thisMerIsBits == false); } // Fail if we see a smaller mer than last time. // if (mer < _thisMer) { char str[1024]; fprintf(stderr, "merylStreamWriter::addMer()-- ERROR: your mer stream isn't sorted increasingly!\n"); fprintf(stderr, "merylStreamWriter::addMer()-- last: %s\n", _thisMer.merToString(str)); fprintf(stderr, "merylStreamWriter::addMer()-- this: %s\n", mer.merToString(str)); exit(1); } // If there was a position given, write it. // if (positions && _POS) for (uint32 i=0; iputBits(positions[i], 32); // If the new mer is the same as the last one just increase the // count. // if (mer == _thisMer) { _thisMerCount += count; return; } // Write thisMer to disk. If the count is zero, we don't write // anything. The count is zero for the first mer (all A) unless we // add that mer, and if the silly user gives us a mer with zero // count. // writeMer(); // If the new mer is in a different bucket from the last mer, write // out some bucket counts. We need a while loop (opposed to just // writing one bucket) because we aren't guaranteed that the mers // are in adjacent buckets. // val = mer.startOfMer(_prefixSize); while (_thisBucket < val) { setIDXnumber(_thisBucketSize); _thisBucketSize = 0; _thisBucket++; } // Remember the new mer for the next time // _thisMer = mer; _thisMerCount = count; } void merylStreamWriter::addMer(uint64 prefix, uint32 prefixBits, uint64 mer, uint32 merBits, uint32 count, uint32 *UNUSED(positions)) { if (_thisMerIsBits == false) { _thisMerIsBits = true; assert(_thisMerIskMer == false); } assert(prefixBits == _prefixSize); assert(prefixBits == _thisMerPreSize); assert(merBits == _thisMerMerSize); assert(prefixBits + merBits == _merSizeInBits); if (((prefix < _thisMerPre)) || ((prefix <= _thisMerPre) && (mer < _thisMerMer))) { assert(0); } if ((prefix == _thisMerPre) && (mer == _thisMerMer)) { _thisMerCount += count; return; } writeMer(); while (_thisBucket < prefix) { setIDXnumber(_thisBucketSize); _thisBucketSize = 0; _thisBucket++; } _thisMerPre = prefix; _thisMerMer = mer; _thisMerCount = count; } canu-1.3/src/meryl/libmeryl.H000066400000000000000000000163611272335422300161640ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/libmeryl/libmeryl.H * * Modifications by: * * Brian P. Walenz from 2003-SEP-08 to 2004-APR-08 * are Copyright 2003-2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-APR-21 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAY-23 to 2014-APR-11 * are Copyright 2005,2007-2008,2012,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2015-JUN-16 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef LIBMERYL_H #define LIBMERYL_H #include "kMer.H" // A merStream reader/writer for meryl mercount data. // // merSize is used to check that the meryl file is the correct size. // If it isn't the code fails. // // The reader returns mers in lexicographic order. No random access. // The writer assumes that mers come in sorted increasingly. // // numUnique the total number of mers with count of one // numDistinct the total number of distinct mers in this file // numTotal the total number of mers in this file class merylStreamReader { public: merylStreamReader(const char *fn, uint32 ms=0); ~merylStreamReader(); kMer &theFMer(void) { return(_thisMer); }; uint64 theCount(void) { return(_thisMerCount); }; bool hasPositions(void) { return(_POS != 0L); }; uint32 *thePositions(void) { return(_thisMerPositions); }; uint32 getPosition(uint32 i) { return(((_POS) && (i < _thisMerCount)) ? _thisMerPositions[i] : ~uint32ZERO); }; uint32 merSize(void) { return(_merSizeInBits >> 1); }; uint32 merCompression(void) { return(_merCompression); }; uint32 prefixSize(void) { return(_prefixSize); }; uint64 numberOfUniqueMers(void) { return(_numUnique); }; uint64 numberOfDistinctMers(void) { return(_numDistinct); }; uint64 numberOfTotalMers(void) { return(_numTotal); }; uint64 histogram(uint32 i) { return((i < _histogramLen) ? _histogram[i] : ~uint64ZERO); }; uint64 histogramLength(void) { return(_histogramLen); }; uint64 histogramMaximumCount(void) { return(_histogramMaxValue); }; bool nextMer(void); bool validMer(void) { return(_validMer); }; private: char _filename[FILENAME_MAX]; bitPackedFile *_IDX; bitPackedFile *_DAT; bitPackedFile *_POS; uint64 getIDXnumber(void) { uint64 n = 1; if (_idxIsPacked) n = _IDX->getNumber(); else n = _IDX->getBits(32); return(n); }; uint64 getDATnumber(void) { uint64 n = 1; if (_datIsPacked) { if (_DAT->getBits(1)) n = _DAT->getNumber() + 2; } else { n = _DAT->getBits(32); } return(n); }; // Why not bool? Seems like the bitPackedFile is incompatible // with bools. uint32 _idxIsPacked; uint32 _datIsPacked; uint32 _posIsPacked; uint32 _merSizeInBits; uint32 _merCompression; uint32 _prefixSize; uint32 _merDataSize; uint64 _thisBucket; uint64 _thisBucketSize; uint64 _numBuckets; kMer _thisMer; uint64 _thisMerCount; uint32 _thisMerPositionsMax; uint32 *_thisMerPositions; uint64 _numUnique; uint64 _numDistinct; uint64 _numTotal; uint64 _histogramPos; // position of the histogram data in IDX uint64 _histogramLen; // number of entries in the histo uint64 _histogramMaxValue; // highest count ever seen uint64 *_histogram; bool _validMer; }; class merylStreamWriter { public: merylStreamWriter(const char *filePrefix, uint32 merSize, // In bases uint32 merComp, // A length, bases uint32 prefixSize, // In bits bool positionsEnabled); ~merylStreamWriter(); void addMer(kMer &mer, uint32 count=1, uint32 *positions=0L); void addMer(uint64 prefix, uint32 prefixBits, uint64 mer, uint32 merBits, uint32 count=1, uint32 *positions=0L); private: void writeMer(void); void setIDXnumber(uint64 n) { if (_idxIsPacked) _IDX->putNumber(n); else _IDX->putBits(n, 32); }; void setDATnumber(uint64 n) { if (_datIsPacked) { if (n == 1) { _DAT->putBits(uint64ZERO, 1); } else { _DAT->putBits(uint64ONE, 1); _DAT->putNumber(n-2); } } else { _DAT->putBits(n, 32); } }; char _filename[FILENAME_MAX]; bitPackedFile *_IDX; bitPackedFile *_DAT; bitPackedFile *_POS; uint32 _idxIsPacked; uint32 _datIsPacked; uint32 _posIsPacked; uint32 _merSizeInBits; uint32 _merCompression; uint32 _prefixSize; uint32 _merDataSize; uint64 _thisBucket; uint64 _thisBucketSize; uint64 _numBuckets; uint64 _numUnique; uint64 _numDistinct; uint64 _numTotal; uint64 _histogramPos; // position of the histogram data in IDX uint64 _histogramLen; // number of entries in the histogram uint64 _histogramMaxValue; // highest count ever seen uint64 *_histogram; bool _thisMerIsBits; bool _thisMerIskMer; kMer _thisMer; uint64 _thisMerPre; uint64 _thisMerMer; uint32 _thisMerPreSize; uint32 _thisMerMerSize; uint64 _thisMerCount; }; #endif // LIBMERYL_H canu-1.3/src/meryl/mapMers-depth.C000066400000000000000000000124511272335422300170420ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/mapMers-depth.C * * Modifications by: * * Brian P. Walenz from 2007-JUN-08 to 2014-APR-11 * are Copyright 2007-2008,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-07 to 2014-DEC-22 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include "bio++.H" #include "seqCache.H" #include "merStream.H" #include "libmeryl.H" #include "existDB.H" #warning this code might not work due to intervalList changes int main(int argc, char **argv) { uint32 merSize = 16; char *merylFile = 0L; char *fastaFile = 0L; bool beVerbose = false; uint32 loCount = 0; uint32 hiCount = ~uint32ZERO; uint32 windowsize = 0; uint32 skipsize = 0; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-m") == 0) { merSize = strtouint32(argv[++arg]); } else if (strcmp(argv[arg], "-mers") == 0) { merylFile = argv[++arg]; } else if (strcmp(argv[arg], "-seq") == 0) { fastaFile = argv[++arg]; } else if (strcmp(argv[arg], "-v") == 0) { beVerbose = true; } else if (strcmp(argv[arg], "-lo") == 0) { loCount = strtouint32(argv[++arg]); } else if (strcmp(argv[arg], "-hi") == 0) { hiCount = strtouint32(argv[++arg]); } else if (strcmp(argv[arg], "-w") == 0) { windowsize = strtouint32(argv[++arg]); } else if (strcmp(argv[arg], "-s") == 0) { skipsize = strtouint32(argv[++arg]); } else { fprintf(stderr, "unknown option '%s'\n", argv[arg]); } arg++; } if ((merylFile == 0L) || (fastaFile == 0L)) { fprintf(stderr, "usage: %s -m mersize -mers mers -seq fasta > output\n", argv[0]); exit(1); } existDB *E = new existDB(merylFile, merSize, existDBcounts | existDBcompressCounts | existDBcompressBuckets, loCount, hiCount); seqCache *F = new seqCache(fastaFile); for (uint32 Sid=0; Sid < F->getNumberOfSequences(); Sid++) { seqInCore *S = F->getSequenceInCore(Sid); merStream *MS = new merStream(new kMerBuilder(merSize), new seqStream(S->sequence(), S->sequenceLength()), true, true); uint32 idlen = 0; intervalDepthRegions *id = new intervalDepthRegions [S->sequenceLength() * 2 + 2]; while (MS->nextMer()) { int32 cnt = (int32)E->count(MS->theFMer()) + (int32)E->count(MS->theRMer()); // Old intervalDepth was to add 'cnt' in the first and subtract 'cnt' in the second. // Then to use the 'ct' field below. // New intervalDepth is the same, but uses the value field. // Count is now the number of intervals that are represented in this block. id[idlen].pos = MS->thePositionInSequence(); id[idlen].change = cnt; id[idlen].open = true; idlen++; id[idlen].pos = MS->thePositionInSequence() + merSize; id[idlen].change = cnt; id[idlen].open = false; idlen++; } intervalList ID(id, idlen); uint32 x = 0; uint32 len = S->sequenceLength(); // Default case, report un-averaged depth at every single location. // if ((windowsize == 0) && (skipsize == 0)) { for (uint32 i=0; i < ID.numberOfIntervals(); i++) { for (; x < ID.lo(i); x++) fprintf(stdout, uint32FMTW(7)"\t"uint32FMTW(6)"\n", x, 0); for (; x < ID.hi(i); x++) fprintf(stdout, uint32FMTW(7)"\t"uint32FMTW(6)"\n", x, ID.value(i)); } for (; x < len; x++) fprintf(stdout, uint32FMTW(7)"\t"uint32FMTW(6)"\n", x, 0); } else { uint32 *depth = new uint32 [len]; for (x=0; x < len; x++) depth[x] = 0; for (uint32 i=0; i < ID.numberOfIntervals(); i++) for (x=ID.lo(i); x < ID.hi(i); x++) depth[x] = ID.count(i); uint32 avedepth = 0; for (x=0; x < windowsize; x++) avedepth += depth[x]; while (x < len) { uint32 avepos = (x - 1) - (windowsize - 1) / 2; if ((avepos % skipsize) == 0) fprintf(stdout, uint32FMT"\t%.4f\n", avepos, (double)avedepth / (double)windowsize); avedepth = avedepth + depth[x] - depth[x-windowsize]; x++; } delete [] depth; } delete [] id; delete MS; delete S; } delete F; delete E; } canu-1.3/src/meryl/mapMers.C000066400000000000000000000157471272335422300157530ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/mapMers.C * * Modifications by: * * Brian P. Walenz from 2006-OCT-18 to 2014-APR-11 * are Copyright 2006-2008,2013-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2015-JUL-22 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include "bio++.H" #include "seqCache.H" #include "merStream.H" #include "libmeryl.H" #include "existDB.H" #define OP_NONE 0 #define OP_STATS 1 #define OP_REGIONS 2 #define OP_DETAILS 3 int main(int argc, char **argv) { uint32 merSize = 16; char *merylFile = 0L; char *fastaFile = 0L; bool beVerbose = false; uint32 loCount = 0; uint32 hiCount = ~uint32ZERO; uint32 operation = OP_NONE; // For OP_STATS uint32 Clen = 0; uint32 Cmax = 4 * 1024 * 1024; uint32 *C = new uint32 [Cmax]; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-m") == 0) { merSize = strtouint32(argv[++arg]); } else if (strcmp(argv[arg], "-mers") == 0) { merylFile = argv[++arg]; } else if (strcmp(argv[arg], "-seq") == 0) { fastaFile = argv[++arg]; } else if (strcmp(argv[arg], "-v") == 0) { beVerbose = true; } else if (strcmp(argv[arg], "-lo") == 0) { loCount = strtouint32(argv[++arg]); } else if (strcmp(argv[arg], "-hi") == 0) { hiCount = strtouint32(argv[++arg]); } else if (strcmp(argv[arg], "-stats") == 0) { operation = OP_STATS; } else if (strcmp(argv[arg], "-regions") == 0) { operation = OP_REGIONS; } else if (strcmp(argv[arg], "-details") == 0) { operation = OP_DETAILS; } else { fprintf(stderr, "unknown option '%s'\n", argv[arg]); } arg++; } if ((operation == OP_NONE) || (merylFile == 0L) || (fastaFile == 0L)) { fprintf(stderr, "usage: %s [-stats | -regions | -details] -m mersize -mers mers -seq fasta > output\n", argv[0]); exit(1); } #if 0 existDB *E = NULL; if (fileExists("junk.existDB")) { fprintf(stderr, "loading from junk.existDB\n"); E = new existDB("junk.existDB"); fprintf(stderr, "loaded\n"); } else { exit(1); E = new existDB(merylFile, merSize, existDBcounts, loCount, hiCount); E->saveState("junk.existDB"); } #endif existDB *E = new existDB(merylFile, merSize, existDBcounts, loCount, hiCount); seqCache *F = new seqCache(fastaFile); fprintf(stderr, "Begin.\n"); for (uint32 Sid=0; Sid < F->getNumberOfSequences(); Sid++) { seqInCore *S = F->getSequenceInCore(Sid); merStream *MS = new merStream(new kMerBuilder(merSize), new seqStream(S->sequence(), S->sequenceLength()), true, true); // with counts, report mean, mode, median, min, max for each frag. if (operation == OP_STATS) { Clen = 0; while (MS->nextMer()) C[Clen++] = E->count(MS->theFMer()) + E->count(MS->theRMer()); uint64 mean = uint64ZERO; uint64 min = ~uint64ZERO; uint64 max = uint64ZERO; uint64 hist[16] = { 0 }; // Histogram values are powers of two, e.g., <=1, <=2, <=4, <=8, <=16, <=32, <=64, <=128, <=256, <=512, <=1024, <=4096, <=8192, <=328768 for (uint32 i=0; i C[i]) min = C[i]; if (max < C[i]) max = C[i]; hist[ logBaseTwo64(C[i]) ]++; } if (Clen > 0) { mean /= Clen; } else { mean = uint64ZERO; min = uint64ZERO; max = uint64ZERO; } fprintf(stdout, "%s\t" uint64FMT"\t"uint64FMT"\t"uint64FMT"\t" uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t" uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\n", S->header(), mean, min, max, hist[ 0], hist[ 1], hist[ 2], hist[ 3], hist[ 4], hist[ 5], hist[ 6], hist[ 7], hist[ 8], hist[ 9], hist[10], hist[11], hist[12], hist[13], hist[14], hist[15]); } // without counts, reports regions with mer coverage. // Orientation tells us nothing, since the mers are probably canonical if (operation == OP_REGIONS) { uint64 beg = ~uint64ZERO; uint64 end = ~uint64ZERO; uint64 pos = ~uint64ZERO; uint64 numCovReg = 0; uint64 lenCovReg = 0; while (MS->nextMer()) { if (E->exists(MS->theFMer()) || E->exists(MS->theRMer())) { pos = MS->thePositionInSequence(); if (beg == ~uint64ZERO) beg = end = pos; if (pos <= end + merSize) { end = pos; } else { fprintf(stdout, "%s\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\n", S->header(), beg, end+merSize, end+merSize - beg); numCovReg++; lenCovReg += end+merSize - beg; beg = end = pos; } } else { fprintf(stdout, "%s\t"uint64FMT"\tuncovered\n", S->header(), MS->thePositionInSequence()); } } if (beg != ~uint64ZERO) fprintf(stdout, "%s\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\n", S->header(), beg, end+merSize, end+merSize - beg); fprintf(stderr, "numCovReg: "uint64FMT"\n", numCovReg); fprintf(stderr, "lenCovReg: "uint64FMT"\n", lenCovReg); } if (operation == OP_DETAILS) { char merString[256]; while (MS->nextMer()) { uint64 beg = MS->thePositionInSequence(); uint64 end = beg + merSize; uint64 fnt = E->count(MS->theFMer()); uint64 rnt = E->count(MS->theRMer()); fprintf(stdout, "%s\t%s\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\t"uint64FMT"\n", S->header(), MS->theFMer().merToString(merString), beg, end, fnt, rnt, fnt + rnt); } } delete MS; delete S; } delete F; delete E; } canu-1.3/src/meryl/maskMers.C000066400000000000000000000425211272335422300161170ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/maskMers.C * * Modifications by: * * Brian P. Walenz from 2008-MAR-31 to 2014-APR-11 * are Copyright 2008,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include #include "bio++.H" #include "seqStream.H" #include "libmeryl.H" #include #define MAX_COVERAGE 51 class mateRescueData { public: mateRescueData() { _mean = 0; _stddev = 0; _coverage = 0; _normal = 0L; _normalZero = 0; }; void init(int32 mean_, int32 stddev_, uint32 coverage_) { _mean = mean_; _stddev = stddev_; _coverage = coverage_; assert(_mean > 3 * _stddev); double a = 1.0 / (_stddev * sqrt(2 * M_PI)); double c = 2 * _stddev * _stddev; int32 b1l = (int32)floor(-3 * _stddev); int32 b1h = (int32)ceil ( 3 * _stddev); _normal = new double [b1h - b1l + 1]; _normalZero = -b1l; for (int32 l=0; l= _numSeq) || (onlySeqIID_ == i)) { fprintf(stderr, "Loading sequence "uint32FMT" of length "uint32FMT"\n", i, _seqLen[i]); _masking[i] = new char [_seqLen[i]]; _repeatID[i] = new uint32 [_seqLen[i]]; //memset(_masking[i], 'g', sizeof(char) * _seqLen[i]); //memset(_repeatID[i], 0, sizeof(uint32) * _seqLen[i]); fread(_masking[i], sizeof(char), _seqLen[i], maskMersFile); fread(_repeatID[i], sizeof(uint32), _seqLen[i], maskMersFile); } else { fseek(maskMersFile, sizeof(char) * _seqLen[i], SEEK_CUR); fseek(maskMersFile, sizeof(uint32) * _seqLen[i], SEEK_CUR); _seqLen[i] = 0; } } fclose(maskMersFile); } void merMaskedSequence::saveMasking(void) { FILE *maskMersFile = fopen(_maskMersName, "w"); fwrite(&_numSeq, sizeof(uint32), 1, maskMersFile); fwrite(&_merSize, sizeof(uint32), 1, maskMersFile); fwrite( _seqLen, sizeof(uint32), _numSeq, maskMersFile); for (uint32 i=0; i<_numSeq; i++) { fwrite(_masking[i], sizeof(char), _seqLen[i], maskMersFile); fwrite(_repeatID[i], sizeof(uint32), _seqLen[i], maskMersFile); } fclose(maskMersFile); } void merMaskedSequence::buildMasking(void) { seqStream *STR = new seqStream(_fastaName); _numSeq = STR->numberOfSequences(); _seqLen = new int32 [_numSeq]; _masking = new char * [_numSeq]; _repeatID = new uint32 * [_numSeq]; _merSize = 0; fprintf(stderr, uint32FMT" sequences in '%s'\n", _numSeq, _fastaName); for (uint32 i=0; i<_numSeq; i++) { _seqLen[i] = STR->lengthOf(i); _masking[i] = new char [_seqLen[i]]; _repeatID[i] = new uint32 [_seqLen[i]]; memset(_masking[i], 'g', sizeof(char) * _seqLen[i]); memset(_repeatID[i], 0, sizeof(uint32) * _seqLen[i]); } // g -> gap in sequence // u -> unique mer // r -> repeat mer // // For all the r's we also need to remember the other locations // that repeat is at. We annotate the map with a repeat id, set if // another copy of the repeat is nearby. merylStreamReader *MS = new merylStreamReader(_merylName); speedCounter *CT = new speedCounter(" Masking mers in sequence: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, true); uint32 rid = 0; _merSize = MS->merSize(); while (MS->nextMer()) { //fprintf(stderr, "mer count="uint64FMT" pos="uint32FMT"\n", MS->theCount(), MS->getPosition(0)); if (MS->theCount() == 1) { uint32 p = MS->getPosition(0); uint32 s = STR->sequenceNumberOfPosition(p); p -= STR->startOf(s); _masking[s][p] = 'u'; } else { std::sort(MS->thePositions(), MS->thePositions() + MS->theCount()); uint32 lastS = ~uint32ZERO; uint32 lastP = 0; rid++; for (uint32 i=0; itheCount(); i++) { uint32 p = MS->getPosition(i); uint32 s = STR->sequenceNumberOfPosition(p); p -= STR->startOf(s); // Always set the masking. _masking[s][p] = 'r'; // If there is a repeat close by, set the repeat ID. if ((s == lastS) && (lastP + 40000 > p)) { _repeatID[s][lastP] = rid; _repeatID[s][p] = rid; } lastS = s; lastP = p; } } CT->tick(); } delete CT; delete MS; delete STR; saveMasking(); } void computeDensity(merMaskedSequence *S, char *outputPrefix) { char outputName[FILENAME_MAX]; FILE *outputFile; uint32 windowSizeMax = 10000; for (uint32 s=0; snumSeq(); s++) { // seqLen == 0 iff that sequence is not loaded. if (S->seqLen(s) == 0) continue; sprintf(outputName, "%s.density.seq"uint32FMTW(02), outputPrefix, s); outputFile = fopen(outputName, "w"); fprintf(stderr, "Starting '%s'\n", outputName); fprintf(outputFile, "#window\tunique\trepeat\tgaps\n"); // Not the most efficient, but good enough for us right now. for (int32 p=0; pseqLen(s); ) { uint32 windowSize = 0; uint32 uniqueSum = 0; uint32 repeatSum = 0; uint32 gapSum = 0; while ((windowSize < windowSizeMax) && (p < S->seqLen(s))) { char m = S->masking(s, p); if (m == 'u') uniqueSum++; if (m == 'g') gapSum++; if (m == 'r') repeatSum++; windowSize++; p++; } fprintf(outputFile, uint32FMT"\t%f\t%f\t%f\n", p - windowSize, (double)uniqueSum / windowSize, (double)repeatSum / windowSize, (double)gapSum / windowSize); } fclose(outputFile); } } // For each 'r' mer, compute the number of 'u' mers // that are within some mean +- stddev range. // // We count for two blocks: // // | <- mean -> | <- mean -> | // ---[block1]---------------mer---------------[block2]--- // // Once we know that, we can compute the probability that // a repeat mer can be rescued. // // p1 = uniq/total -- for 1 X coverage // pn = 1 - (1-p1)^n -- for n X coverage void computeMateRescue(merMaskedSequence *S, char *outputPrefix, mateRescueData *lib, uint32 libLen) { char outputName[FILENAME_MAX]; FILE *outputFile; FILE *outputData; uint32 closeRepeatsLen = 0; uint32 closeRepeatsMax = 80000; int32 *closeRepeats = new int32 [closeRepeatsMax]; speedCounter *CT = new speedCounter(" Examining repeats: %7.2f Kbases -- %5.2f Kbases/second\r", 1000.0, 0x1ffff, true); uint32 totalDepth = 0; for (uint32 l=0; lnumSeq(); s++) { // seqLen == 0 iff that sequence is not loaded. if (S->seqLen(s) == 0) continue; fprintf(stderr, "Starting sequence "uint32FMT"\n", s); sprintf(outputName, "%s.mateRescue.seq"uint32FMTW(02)".out", outputPrefix, s); outputFile = fopen(outputName, "w"); sprintf(outputName, "%s.mateRescue.seq"uint32FMTW(02)".dat", outputPrefix, s); outputData = fopen(outputName, "w"); double numRR[MAX_COVERAGE] = {0}; // num repeats rescued (expected) for [] X coverage double numNR[MAX_COVERAGE] = {0}; // num repeats nonrescuable (expected) for [] X coverage uint32 numRT = 0; // num repeats total for (int32 p=0; pseqLen(s); p++) { CT->tick(); double pRtot = 0.0; double pFtot = 0.0; if ((S->masking(s, p) != 'g') && (S->masking(s, p) != 'u') && (S->masking(s, p) != 'r')) fprintf(stderr, "INVALID MASKING - got %d = %c\n", S->masking(s, p), S->masking(s, p)); if (S->masking(s, p) == 'r') { numRT++; // Index over x-coverage in libraries. MUST BE 1. uint32 ridx = 1; for (uint32 l=0; lrepeatID(s, p) > 0) { int32 pl = (int32)floor(p - 3 * stddev); int32 ph = (int32)ceil (p + 3 * stddev); if (pl < 0) pl = 0; if (ph > S->seqLen(s)) ph = S->seqLen(s); for (int32 pi=pl; pirepeatID(s, pi) == S->repeatID(s, p)) && (pi != p)) closeRepeats[closeRepeatsLen++] = pi; } int32 b1l = (int32)floor(p - mean - 3 * stddev); int32 b1h = (int32)ceil (p - mean + 3 * stddev); int32 b2l = (int32)floor(p + mean - 3 * stddev); int32 b2h = (int32)ceil (p + mean + 3 * stddev); if (b1l < 0) b1l = 0; if (b1h < 0) b1h = 0; if (b1h > S->seqLen(s)) b1h = S->seqLen(s); if (b2l < 0) b2l = 0; if (b2h > S->seqLen(s)) b2h = S->seqLen(s); if (b2l > S->seqLen(s)) b2l = S->seqLen(s); //fprintf(stderr, "b1: %d-%d b2:%d-%d\n", b1l, b1h, b2l, b2h); // probability we can rescue this repeat with this mate pair double pRescue = 0.0; double pFailed = 0.0; if (closeRepeatsLen == 0) { // No close repeats, use the fast method. for (int32 b=b1l; bmasking(s, b) == 'u') pRescue += lib[l].normal(b - p + mean); } for (int32 b=b2l; bmasking(s, b) == 'u') pRescue += lib[l].normal(b - p - mean); } } else { // Close repeats, gotta be slow. for (int32 b=b1l; bmasking(s, b) == 'u') { int32 mrl = b + mean - 3 * stddev; int32 mrh = b + mean + 3 * stddev; bool rescuable = true; for (uint32 cri=0; rescuable && crimasking(s, b) == 'u') { int32 mrl = b - mean - 3 * stddev; int32 mrh = b - mean + 3 * stddev; bool rescuable = true; for (uint32 cri=0; rescuable && crimerSize(), numRT, numRR[x], numNR[x], x, lib[l].mean(), lib[l].stddev()); n++; if (n >= lib[l].coverage()) { l++; n = 0; } } fclose(outputFile); fclose(outputData); } delete CT; } int main(int argc, char **argv) { char *merylName = 0L; char *fastaName = 0L; char *outputPrefix = 0L; uint32 onlySeqIID = ~uint32ZERO; bool doDensity = false; bool doRescue = false; mateRescueData lib[MAX_COVERAGE]; uint32 libLen = 0; int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-mers") == 0) { merylName = argv[++arg]; } else if (strcmp(argv[arg], "-seq") == 0) { fastaName = argv[++arg]; } else if (strcmp(argv[arg], "-only") == 0) { onlySeqIID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-output") == 0) { outputPrefix = argv[++arg]; } else if (strcmp(argv[arg], "-d") == 0) { doDensity = true; } else if (strcmp(argv[arg], "-r") == 0) { if (atoi(argv[arg+3]) > 0) { doRescue = true; lib[libLen++].init(atoi(argv[arg+1]), atoi(argv[arg+2]), atoi(argv[arg+3])); } arg += 3; } else { fprintf(stderr, "unknown option '%s'\n", argv[arg]); err++; } arg++; } if ((err) || (merylName == 0L) || (fastaName == 0L) || (outputPrefix == 0L)) { fprintf(stderr, "usage: %s -mers mers -seq fasta -output prefix [-d] [-r mean stddev coverage]\n", argv[0]); exit(1); } merMaskedSequence *S = new merMaskedSequence(fastaName, merylName, onlySeqIID); if (doDensity) computeDensity(S, outputPrefix); if (doRescue) computeMateRescue(S, outputPrefix, lib, libLen); return(0); } canu-1.3/src/meryl/meryl-args.C000066400000000000000000000541731272335422300164250ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/args.C * * Modifications by: * * Brian P. Walenz from 2004-MAR-31 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-APR-05 to 2004-JUL-01 * are Copyright 2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAR-20 to 2014-APR-11 * are Copyright 2005-2011,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2015-MAY-29 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "meryl.H" // Some string handling utilities. // bool writeString(const char *str, FILE *F) { errno = 0; uint32 len = 0; if (str) { len = (uint32)strlen(str) + 1; fwrite(&len, sizeof(uint32), 1, F); fwrite( str, sizeof(char), len, F); } else { fwrite(&len, sizeof(uint32), 1, F); } if (errno) { fprintf(stderr, "writeString()-- Failed to write string of length "F_U32": %s\n", len, strerror(errno)); fprintf(stderr, "writeString()-- First 80 bytes of string is:\n"); fprintf(stderr, "%80.80s\n", str); return(false); } return(true); } char* readString(FILE *F) { errno = 0; uint32 len = 0; fread(&len, sizeof(uint32), 1, F); if (errno) { fprintf(stderr, "readString()-- Failed to read string: %s\n", strerror(errno)); exit(1); } char *str = 0L; if (len > 0) { str = new char [len]; fread(str, sizeof(char), len, F); if (errno) { fprintf(stderr, "readString()-- Failed to read string: %s\n", strerror(errno)); exit(1); } } return(str); } char* duplString(char *str) { char *dupstr = 0L; if (str) { uint32 len = (uint32)strlen(str); dupstr = new char [len+1]; strcpy(dupstr, str); } return(dupstr); } void merylArgs::usage(void) { fprintf(stderr, "usage: %s [personality] [global options] [options]\n", execName); fprintf(stderr, "\n"); fprintf(stderr, "where personality is:\n"); fprintf(stderr, " -P -- compute parameters\n"); fprintf(stderr, " -B -- build table\n"); fprintf(stderr, " -S -- scan table\n"); fprintf(stderr, " -M -- \"math\" operations\n"); fprintf(stderr, " -D -- dump table\n"); fprintf(stderr, "\n"); fprintf(stderr, "-P: Given a sequence file (-s) or an upper limit on the\n"); fprintf(stderr, " number of mers in the file (-n), compute the table size\n"); fprintf(stderr, " (-t in build) to minimize the memory usage.\n"); fprintf(stderr, " -m # (size of a mer; required)\n"); fprintf(stderr, " -c # (homopolymer compression; optional)\n"); fprintf(stderr, " -p (enable positions)\n"); fprintf(stderr, " -s seq.fasta (seq.fasta is scanned to determine the number of mers)\n"); fprintf(stderr, " -n # (compute params assuming file with this many mers in it)\n"); fprintf(stderr, "\n"); fprintf(stderr, " Only one of -s, -n need to be specified. If both are given\n"); fprintf(stderr, " -s takes priority.\n"); fprintf(stderr, "\n"); fprintf(stderr, "\n"); fprintf(stderr, "-B: Given a sequence file (-s) and lots of parameters, compute\n"); fprintf(stderr, " the mer-count tables. By default, both strands are processed.\n"); fprintf(stderr, " -f (only build for the forward strand)\n"); fprintf(stderr, " -r (only build for the reverse strand)\n"); fprintf(stderr, " -C (use canonical mers, assumes both strands)\n"); fprintf(stderr, " -L # (DON'T save mers that occur less than # times)\n"); fprintf(stderr, " -U # (DON'T save mers that occur more than # times)\n"); fprintf(stderr, " -m # (size of a mer; required)\n"); fprintf(stderr, " -c # (homopolymer compression; optional)\n"); fprintf(stderr, " -p (enable positions)\n"); fprintf(stderr, " -s seq.fasta (sequence to build the table for)\n"); fprintf(stderr, " -o tblprefix (output table prefix)\n"); fprintf(stderr, " -v (entertain the user)\n"); fprintf(stderr, "\n"); fprintf(stderr, " By default, the computation is done as one large sequential process.\n"); fprintf(stderr, " Multi-threaded operation is possible, at additional memory expense, as\n"); fprintf(stderr, " is segmented operation, at additional I/O expense.\n"); fprintf(stderr, "\n"); fprintf(stderr, " Threaded operation: Split the counting in to n almost-equally sized\n"); fprintf(stderr, " pieces. This uses an extra h MB (from -P) per thread.\n"); fprintf(stderr, " -threads n (use n threads to build)\n"); fprintf(stderr, "\n"); fprintf(stderr, " Segmented, sequential operation: Split the counting into pieces that\n"); fprintf(stderr, " will fit into no more than m MB of memory, or into n equal sized pieces.\n"); fprintf(stderr, " Each piece is computed sequentially, and the results are merged at the end.\n"); fprintf(stderr, " Only one of -memory and -segments is needed.\n"); fprintf(stderr, " -memory mMB (use at most m MB of memory per segment)\n"); fprintf(stderr, " -segments n (use n segments)\n"); fprintf(stderr, "\n"); fprintf(stderr, " Segmented, batched operation: Same as sequential, except this allows\n"); fprintf(stderr, " each segment to be manually executed in parallel.\n"); fprintf(stderr, " Only one of -memory and -segments is needed.\n"); fprintf(stderr, " -memory mMB (use at most m MB of memory per segment)\n"); fprintf(stderr, " -segments n (use n segments)\n"); fprintf(stderr, " -configbatch (create the batches)\n"); fprintf(stderr, " -countbatch n (run batch number n)\n"); fprintf(stderr, " -mergebatch (merge the batches)\n"); fprintf(stderr, " Initialize the compute with -configbatch, which needs all the build options.\n"); fprintf(stderr, " Execute all -countbatch jobs, then -mergebatch to complete.\n"); fprintf(stderr, " meryl -configbatch -B [options] -o file\n"); fprintf(stderr, " meryl -countbatch 0 -o file\n"); fprintf(stderr, " meryl -countbatch 1 -o file\n"); fprintf(stderr, " ...\n"); fprintf(stderr, " meryl -countbatch N -o file\n"); fprintf(stderr, " meryl -mergebatch N -o file\n"); fprintf(stderr, " Batched mode can run on the grid.\n"); fprintf(stderr, " -sge jobname unique job name for this execution. Meryl will submit\n"); fprintf(stderr, " jobs with name mpjobname, ncjobname, nmjobname, for\n"); fprintf(stderr, " phases prepare, count and merge.\n"); fprintf(stderr, " -sgebuild \"options\" any additional options to sge, e.g.,\n"); fprintf(stderr, " -sgemerge \"options\" \"-p -153 -pe thread 2 -A merylaccount\"\n"); fprintf(stderr, " N.B. - -N will be ignored\n"); fprintf(stderr, " N.B. - be sure to quote the options\n"); fprintf(stderr, "\n"); fprintf(stderr, "-M: Given a list of tables, perform a math, logical or threshold operation.\n"); fprintf(stderr, " Unless specified, all operations take any number of databases.\n"); fprintf(stderr, "\n"); fprintf(stderr, " Math operations are:\n"); fprintf(stderr, " min count is the minimum count for all databases. If the mer\n"); fprintf(stderr, " does NOT exist in all databases, the mer has a zero count, and\n"); fprintf(stderr, " is NOT in the output.\n"); fprintf(stderr, " minexist count is the minimum count for all databases that contain the mer\n"); fprintf(stderr, " max count is the maximum count for all databases\n"); fprintf(stderr, " add count is sum of the counts for all databases\n"); fprintf(stderr, " sub count is the first minus the second (binary only)\n"); fprintf(stderr, " abs count is the absolute value of the first minus the second (binary only)\n"); fprintf(stderr, "\n"); fprintf(stderr, " Logical operations are:\n"); fprintf(stderr, " and outputs mer iff it exists in all databases\n"); fprintf(stderr, " nand outputs mer iff it exists in at least one, but not all, databases\n"); fprintf(stderr, " or outputs mer iff it exists in at least one database\n"); fprintf(stderr, " xor outputs mer iff it exists in an odd number of databases\n"); fprintf(stderr, "\n"); fprintf(stderr, " Threshold operations are:\n"); fprintf(stderr, " lessthan x outputs mer iff it has count < x\n"); fprintf(stderr, " lessthanorequal x outputs mer iff it has count <= x\n"); fprintf(stderr, " greaterthan x outputs mer iff it has count > x\n"); fprintf(stderr, " greaterthanorequal x outputs mer iff it has count >= x\n"); fprintf(stderr, " equal x outputs mer iff it has count == x\n"); fprintf(stderr, " Threshold operations work on exactly one database.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -s tblprefix (use tblprefix as a database)\n"); fprintf(stderr, " -o tblprefix (create this output)\n"); fprintf(stderr, " -v (entertain the user)\n"); fprintf(stderr, "\n"); fprintf(stderr, " NOTE: Multiple tables are specified with multiple -s switches; e.g.:\n"); fprintf(stderr, " %s -M add -s 1 -s 2 -s 3 -s 4 -o all\n", execName); fprintf(stderr, " NOTE: It is NOT possible to specify more than one operation:\n"); fprintf(stderr, " %s -M add -s 1 -s 2 -sub -s 3\n", execName); fprintf(stderr, " will NOT work.\n"); fprintf(stderr, "\n"); fprintf(stderr, "\n"); fprintf(stderr, "-D: Dump the table (not all of these work).\n"); fprintf(stderr, "\n"); fprintf(stderr, " -Dd Dump a histogram of the distance between the same mers.\n"); fprintf(stderr, " -Dt Dump mers >= a threshold. Use -n to specify the threshold.\n"); fprintf(stderr, " -Dc Count the number of mers, distinct mers and unique mers.\n"); fprintf(stderr, " -Dh Dump (to stdout) a histogram of mer counts.\n"); fprintf(stderr, " -s Read the count table from here (leave off the .mcdat or .mcidx).\n"); fprintf(stderr, "\n"); fprintf(stderr, "\n"); } void merylArgs::clear(void) { execName = 0L; options = 0L; beVerbose = false; doForward = true; doReverse = false; doCanonical = false; inputFile = 0L; outputFile = 0L; queryFile = 0L; merSize = 20; merComp = 0; positionsEnabled = false; numMersEstimated = 0; numMersActual = 0; numBasesActual = 0; mersPerBatch = 0; basesPerBatch = 0; numBuckets = 0; numBuckets_log2 = 0; merDataWidth = 0; merDataMask = uint64ZERO; bucketPointerWidth = 0; numThreads = 0; memoryLimit = 0; segmentLimit = 0; configBatch = false; countBatch = false; mergeBatch = false; batchNumber = 0; sgeJobName = 0L; sgeBuildOpt = 0L; sgeMergeOpt = 0L; isOnGrid = false; lowCount = 0; highCount = ~lowCount; desiredCount = 0; outputCount = 0; outputAll = 0; outputPosition = 0; mergeFilesMax = 0; mergeFilesLen = 0; mergeFiles = 0L; personality = 0; } merylArgs::merylArgs(int argc, char **argv) { clear(); execName = duplString(argv[0]); if (argc == 1) { usage(); exit(1); } // Count how many '-s' switches there are, then allocate space // for them in mergeFiles. We also sum the length of all options, // so we can copy them into an 'options' string used when we // resubmit to the grid. // uint32 optionsLen = 0; for (int arg=1; arg < argc; arg++) { optionsLen += strlen(argv[arg]) + 1; if (strcmp(argv[arg], "-s") == 0) mergeFilesMax++; } mergeFiles = new char * [mergeFilesMax]; options = new char [2 * optionsLen + 1]; options[0] = 0; bool fail = false; char *optptr = options; for (int arg=1; arg < argc; arg++) { if (arg > 1) *optptr++ = ' '; // Arg! If the arg has spaces or other stuff that the shell // needs escaped we need to escape them again. So, we copy byte // by byte and insert escapes at the right points. for (char *op=argv[arg]; *op; op++, optptr++) { if (isspace(*op) || !isalnum(*op)) if ((*op != '-') && (*op != '_') && (*op != '.') && (*op != '/')) *optptr++ = '\\'; *optptr = *op; } //strcat(options, argv[arg]); } // Parse the options // for (int arg=1; arg < argc; arg++) { if (strncmp(argv[arg], "-V", 2) == 0) { fprintf(stdout, "meryl the Mighty Mer Counter version (no version)\n"); exit(0); } else if (strcmp(argv[arg], "-m") == 0) { arg++; merSize = strtouint32(argv[arg]); } else if (strcmp(argv[arg], "-c") == 0) { arg++; merComp = strtouint32(argv[arg]); } else if (strcmp(argv[arg], "-p") == 0) { positionsEnabled = true; } else if (strcmp(argv[arg], "-s") == 0) { arg++; delete [] inputFile; inputFile = duplString(argv[arg]); mergeFiles[mergeFilesLen++] = duplString(argv[arg]); } else if (strcmp(argv[arg], "-n") == 0) { arg++; numMersEstimated = strtouint64(argv[arg]); } else if (strcmp(argv[arg], "-f") == 0) { doForward = true; doReverse = false; doCanonical = false; } else if (strcmp(argv[arg], "-r") == 0) { doForward = false; doReverse = true; doCanonical = false; } else if (strcmp(argv[arg], "-C") == 0) { doForward = false; doReverse = false; doCanonical = true; } else if (strcmp(argv[arg], "-L") == 0) { arg++; lowCount = strtouint32(argv[arg]); } else if (strcmp(argv[arg], "-U") == 0) { arg++; highCount = strtouint32(argv[arg]); } else if (strcmp(argv[arg], "-o") == 0) { arg++; delete [] outputFile; outputFile = duplString(argv[arg]); } else if (strcmp(argv[arg], "-v") == 0) { beVerbose = true; } else if (strcmp(argv[arg], "-P") == 0) { personality = 'P'; } else if (strcmp(argv[arg], "-B") == 0) { personality = 'B'; } else if (strcmp(argv[arg], "-S") == 0) { personality = 'S'; } else if (strcmp(argv[arg], "-M") == 0) { arg++; if (strcmp(argv[arg], "merge") == 0) { personality = PERSONALITY_MERGE; } else if (strcmp(argv[arg], "min") == 0) { personality = PERSONALITY_MIN; } else if (strcmp(argv[arg], "minexist") == 0) { personality = PERSONALITY_MINEXIST; } else if (strcmp(argv[arg], "max") == 0) { personality = PERSONALITY_MAX; } else if (strcmp(argv[arg], "maxexist") == 0) { personality = PERSONALITY_MAXEXIST; } else if (strcmp(argv[arg], "add") == 0) { personality = PERSONALITY_ADD; } else if (strcmp(argv[arg], "sub") == 0) { personality = PERSONALITY_SUB; } else if (strcmp(argv[arg], "abs") == 0) { personality = PERSONALITY_ABS; } else if (strcmp(argv[arg], "divide") == 0) { personality = PERSONALITY_DIVIDE; } else if (strcmp(argv[arg], "and") == 0) { personality = PERSONALITY_AND; } else if (strcmp(argv[arg], "nand") == 0) { personality = PERSONALITY_NAND; } else if (strcmp(argv[arg], "or") == 0) { personality = PERSONALITY_OR; } else if (strcmp(argv[arg], "xor") == 0) { personality = PERSONALITY_XOR; } else if (strcmp(argv[arg], "lessthan") == 0) { personality = PERSONALITY_LEQ; arg++; desiredCount = strtouint32(argv[arg]) - 1; } else if (strcmp(argv[arg], "lessthanorequal") == 0) { personality = PERSONALITY_LEQ; arg++; desiredCount = strtouint32(argv[arg]); } else if (strcmp(argv[arg], "greaterthan") == 0) { personality = PERSONALITY_GEQ; arg++; desiredCount = strtouint32(argv[arg]) + 1; } else if (strcmp(argv[arg], "greaterthanorequal") == 0) { personality = PERSONALITY_GEQ; arg++; desiredCount = strtouint32(argv[arg]); } else if (strcmp(argv[arg], "equal") == 0) { personality = PERSONALITY_EQ; arg++; desiredCount = strtouint32(argv[arg]); } else { fprintf(stderr, "ERROR: unknown math personality %s\n", argv[arg]); exit(1); } } else if (strcmp(argv[arg], "-Dd") == 0) { personality = 'd'; } else if (strcmp(argv[arg], "-Dt") == 0) { personality = 't'; } else if (strcmp(argv[arg], "-Dp") == 0) { personality = 'p'; } else if (strcmp(argv[arg], "-Dc") == 0) { personality = 'c'; } else if (strcmp(argv[arg], "-Dh") == 0) { personality = 'h'; } else if (strcmp(argv[arg], "-memory") == 0) { arg++; memoryLimit = strtouint64(argv[arg]) * 1024 * 1024; } else if (strcmp(argv[arg], "-segments") == 0) { arg++; segmentLimit = strtouint64(argv[arg]); } else if (strcmp(argv[arg], "-threads") == 0) { arg++; numThreads = strtouint32(argv[arg]); } else if (strcmp(argv[arg], "-configbatch") == 0) { personality = 'B'; configBatch = true; countBatch = false; mergeBatch = false; batchNumber = uint32ZERO; } else if (strcmp(argv[arg], "-countbatch") == 0) { arg++; personality = 'B'; configBatch = false; countBatch = true; mergeBatch = false; batchNumber = strtouint32(argv[arg]); } else if (strcmp(argv[arg], "-mergebatch") == 0) { personality = 'B'; configBatch = false; countBatch = false; mergeBatch = true; batchNumber = uint32ZERO; } else if (strcmp(argv[arg], "-sge") == 0) { sgeJobName = argv[++arg]; } else if (strcmp(argv[arg], "-sgebuild") == 0) { sgeBuildOpt = argv[++arg]; } else if (strcmp(argv[arg], "-sgemerge") == 0) { sgeMergeOpt = argv[++arg]; } else if (strcmp(argv[arg], "-forcebuild") == 0) { isOnGrid = true; } else { fprintf(stderr, "Unknown option '%s'.\n", argv[arg]); fail = true; } } // Using threads is only useful if we are not a batch. // if ((numThreads > 0) && (configBatch || countBatch || mergeBatch)) { if (configBatch) fprintf(stderr, "WARNING: -threads has no effect with -configbatch, disabled.\n"); if (countBatch) fprintf(stderr, "WARNING: -threads has no effect with -countbatch, disabled.\n"); if (mergeBatch) fprintf(stderr, "WARNING: -threads has no effect with -mergebatch, disabled.\n"); numThreads = 0; } // SGE is not useful unless we are in batch mode. // if (sgeJobName && !configBatch && !countBatch && !mergeBatch) { fprintf(stderr, "ERROR: -sge not useful unless in batch mode (replace -B with -configbatch)\n"); exit(1); } if (fail) exit(1); } merylArgs::merylArgs(const char *prefix) { clear(); char *filename = new char [strlen(prefix) + 17]; sprintf(filename, "%s.merylArgs", prefix); errno = 0; FILE *F = fopen(filename, "rb"); if (errno) { fprintf(stderr, "merylArgs::readConfig()-- Failed to open '%s': %s\n", filename, strerror(errno)); exit(1); } char magic[17] = {0}; fread(magic, sizeof(char), 16, F); if (strncmp(magic, "merylBatcherv02", 16) != 0) { fprintf(stderr, "merylArgs::readConfig()-- '%s' doesn't appear to be a merylArgs file.\n", filename); exit(1); } // Load the config, then reset the pointers. fread(this, sizeof(merylArgs), 1, F); execName = readString(F); options = 0L; inputFile = readString(F); outputFile = readString(F); queryFile = 0L; sgeJobName = readString(F); sgeBuildOpt = readString(F); sgeMergeOpt = readString(F); mergeFiles = new char* [mergeFilesLen]; for (uint32 i=0; i #include #include #include #include "meryl.H" #include "libmeryl.H" void binaryOperations(merylArgs *args) { if (args->mergeFilesLen != 2) { fprintf(stderr, "ERROR - must have exactly two files!\n"); exit(1); } if (args->outputFile == 0L) { fprintf(stderr, "ERROR - no output file specified.\n"); exit(1); } if ((args->personality != PERSONALITY_SUB) && (args->personality != PERSONALITY_ABS) && (args->personality != PERSONALITY_DIVIDE)) { fprintf(stderr, "ERROR - only personalities sub and abs\n"); fprintf(stderr, "ERROR - are supported in binaryOperations().\n"); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); } // Open the input files, read in the first mer // merylStreamReader *A = new merylStreamReader(args->mergeFiles[0]); merylStreamReader *B = new merylStreamReader(args->mergeFiles[1]); A->nextMer(); B->nextMer(); // Make sure that the mersizes agree, and pick a prefix size for // the output // if (A->merSize() != B->merSize()) { fprintf(stderr, "ERROR - mersizes are different!\n"); fprintf(stderr, "ERROR - mersize of '%s' is "F_U32"\n", args->mergeFiles[0], A->merSize()); fprintf(stderr, "ERROR - mersize of '%s' is "F_U32"\n", args->mergeFiles[1], B->merSize()); exit(1); } // Open the output file, using the larger of the two prefix sizes // merylStreamWriter *W = new merylStreamWriter(args->outputFile, A->merSize(), A->merCompression(), (A->prefixSize() > B->prefixSize()) ? A->prefixSize() : B->prefixSize(), A->hasPositions()); // SUB - report A - B // ABS - report the absolute difference between the two files // // These two operations are very similar (SUB was derived from ABS), so // any bug found in one is probably in the other. // kMer Amer; uint32 Acnt = uint32ZERO; kMer Bmer; uint32 Bcnt = uint32ZERO; switch (args->personality) { case PERSONALITY_SUB: while (A->validMer() || B->validMer()) { Amer = A->theFMer(); Acnt = A->theCount(); Bmer = B->theFMer(); Bcnt = B->theCount(); // If the A stream is all out of mers, set Amer to be the // same as Bmer, and set Acnt to zero. Similar for B. // if (!A->validMer()) { Amer = Bmer; Acnt = uint32ZERO; } if (!B->validMer()) { Bmer = Amer; Bcnt = uint32ZERO; } //fprintf(stderr, "sub A="uint64HEX" B="uint64HEX"\n", Amer, Bmer); if (Amer == Bmer) { W->addMer(Amer, (Acnt > Bcnt) ? Acnt - Bcnt : 0); A->nextMer(); B->nextMer(); } else if (Amer < Bmer) { W->addMer(Amer, Acnt); A->nextMer(); } else { B->nextMer(); } } break; case PERSONALITY_ABS: while (A->validMer() || B->validMer()) { Amer = A->theFMer(); Acnt = A->theCount(); Bmer = B->theFMer(); Bcnt = B->theCount(); // If the A stream is all out of mers, set Amer to be the // same as Bmer, and set Acnt to zero. Similar for B. // if (!A->validMer()) { Amer = Bmer; Acnt = uint32ZERO; } if (!B->validMer()) { Bmer = Amer; Bcnt = uint32ZERO; } if (Amer == Bmer) { W->addMer(Amer, (Acnt > Bcnt) ? Acnt - Bcnt : Bcnt - Acnt); A->nextMer(); B->nextMer(); } else if (Amer < Bmer) { W->addMer(Amer, Acnt); A->nextMer(); } else { W->addMer(Bmer, Bcnt); B->nextMer(); } } break; case PERSONALITY_DIVIDE: while (A->validMer() || B->validMer()) { Amer = A->theFMer(); Acnt = A->theCount(); Bmer = B->theFMer(); Bcnt = B->theCount(); // If the A stream is all out of mers, set Amer to be the // same as Bmer, and set Acnt to zero. Similar for B. // if (!A->validMer()) { Amer = Bmer; Acnt = uint32ZERO; } if (!B->validMer()) { Bmer = Amer; Bcnt = uint32ZERO; } if (Amer == Bmer) { if ((Acnt > 0) && (Bcnt > 0)) { double d = 1000.0 * (double)Acnt / (double)Bcnt; if (d > 4096.0 * 1024.0 * 1024.0) d = 4096.0 * 1024.0 * 1024.0; W->addMer(Amer, (uint32)floor(d)); } A->nextMer(); B->nextMer(); } else if (Amer < Bmer) { A->nextMer(); } else { B->nextMer(); } } break; } delete A; delete B; delete W; } canu-1.3/src/meryl/meryl-build-threads.C000066400000000000000000000061501272335422300202100ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/build-threads.C * * Modifications by: * * Brian P. Walenz from 2004-APR-13 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2006-MAY-14 to 2014-APR-11 * are Copyright 2006,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "meryl.H" void runSegment(merylArgs *args, uint64 segment); pthread_mutex_t segmentMutex; uint64 segmentNext; uint64 segmentMax; uint32 *segmentDone; void* buildThread(void *U) { uint64 segment = uint32ZERO; merylArgs *args = (merylArgs *)U; while (segment < segmentMax) { pthread_mutex_lock(&segmentMutex); segment = segmentNext++; pthread_mutex_unlock(&segmentMutex); if (segment < segmentMax) { runSegment(args, segment); segmentDone[segment]++; } } if (args->beVerbose) fprintf(stderr, "Thread exits.\n"); return(0L); } void runThreaded(merylArgs *args) { // Clear stuff // segmentNext = uint64ZERO; segmentMax = args->segmentLimit; segmentDone = new uint32 [segmentMax]; for (uint64 s=0; snumThreads; i++) pthread_create(&threadID, &threadAttr, buildThread, (void *)args); // Wait for the threads to complete // struct timespec shortNap; shortNap.tv_sec = 1; shortNap.tv_nsec = 0; uint64 s=0; while (s < segmentMax) { if (segmentDone[s] == 0) nanosleep(&shortNap, 0L); else s++; } if (args->beVerbose) fprintf(stderr, "Threads all done, cleaning up.\n"); // Cleanup // pthread_attr_destroy(&threadAttr); pthread_mutex_destroy(&segmentMutex); delete [] segmentDone; } canu-1.3/src/meryl/meryl-build.C000066400000000000000000000662541272335422300165730ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/build.C * * Modifications by: * * Brian P. Walenz from 2003-JAN-02 to 2004-APR-08 * are Copyright 2003-2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Clark Mobarry on 2004-FEB-12 * are Copyright 2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-MAR-23 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAR-12 to 2014-APR-11 * are Copyright 2005-2009,2011-2012,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2015-JUL-01 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "meryl.H" #include "seqStream.H" #include "merStream.H" #include "speedCounter.H" void runThreaded(merylArgs *args); // You probably want this to be the same as KMER_WORDS, but in rare // cases, it can be less. // #define SORTED_LIST_WIDTH KMER_WORDS // to make the sorted list be wider, we also need to store wide // things in the bitpackedarray buckets. probably easy (do multiple // adds of data, each at most 64 bits) but not braindead. #if SORTED_LIST_WIDTH == 1 class sortedList_t { public: uint64 _w; uint32 _p; bool operator<(sortedList_t &that) { return(_w < that._w); }; bool operator>=(sortedList_t &that) { return(_w >= that._w); }; sortedList_t &operator=(sortedList_t &that) { _w = that._w; _p = that._p; return(*this); }; }; #else class sortedList_t { public: uint64 _w[SORTED_LIST_WIDTH]; uint32 _p; bool operator<(sortedList_t &that) { for (uint32 i=SORTED_LIST_WIDTH; i--; ) { if (_w[i] < that._w[i]) return(true); if (_w[i] > that._w[i]) return(false); } return(false); }; bool operator>=(sortedList_t &that) { for (uint32 i=SORTED_LIST_WIDTH; i--; ) { if (_w[i] > that._w[i]) return(true); if (_w[i] < that._w[i]) return(false); } return(true); }; sortedList_t &operator=(sortedList_t &that) { for (uint32 i=SORTED_LIST_WIDTH; i--; ) _w[i] = that._w[i]; _p = that._p; return(*this); }; }; #endif void adjustHeap(sortedList_t *M, int64 i, int64 n) { sortedList_t m = M[i]; int64 j = (i << 1) + 1; // let j be the left child while (j < n) { if (j= M[j]) // a position for M[i] has been found break; M[(j-1)/2] = M[j]; // Move larger child up a level j = (j << 1) + 1; } M[(j-1)/2] = m; } void submitPrepareBatch(merylArgs *args) { FILE *F; char nam[1024]; char cmd[1024]; sprintf(nam, "%s-prepare.sh", args->outputFile); errno = 0; F = fopen(nam, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", nam, strerror(errno)), exit(1); fprintf(F, "#!/bin/sh\n\n"); fprintf(F, ". $SGE_ROOT/$SGE_CELL/common/settings.sh\n"); fprintf(F, "%s -forcebuild %s\n", args->execName, args->options); fclose(F); if (args->sgeMergeOpt) sprintf(cmd, "qsub -cwd -b n -j y -o %s-prepare.err %s -N mp%s %s-prepare.sh", args->outputFile, args->sgeMergeOpt, args->sgeJobName, args->outputFile); else sprintf(cmd, "qsub -cwd -b n -j y -o %s-prepare.err -N mp%s %s-prepare.sh", args->outputFile, args->sgeJobName, args->outputFile); fprintf(stderr, "%s\n", cmd); if (system(cmd)) fprintf(stderr, "%s\nFailed to execute qsub command: %s\n", cmd, strerror(errno)), exit(1); } void submitCountBatches(merylArgs *args) { FILE *F; char nam[1024]; char cmd[1024]; sprintf(nam, "%s-count.sh", args->outputFile); errno = 0; F = fopen(nam, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", nam, strerror(errno)), exit(1); fprintf(F, "#!/bin/sh\n\n"); fprintf(F, ". $SGE_ROOT/$SGE_CELL/common/settings.sh\n"); fprintf(F, "batchnum=`expr $SGE_TASK_ID - 1`\n"); fprintf(F, "%s -v -countbatch $batchnum -o %s\n", args->execName, args->outputFile); fclose(F); if (args->sgeBuildOpt) sprintf(cmd, "qsub -t 1-"F_U64" -cwd -b n -j y -o %s-count-\\$TASK_ID.err %s -N mc%s %s-count.sh", args->segmentLimit, args->outputFile, args->sgeBuildOpt, args->sgeJobName, args->outputFile); else sprintf(cmd, "qsub -t 1-"F_U64" -cwd -b n -j y -o %s-count-\\$TASK_ID.err -N mc%s %s-count.sh", args->segmentLimit, args->outputFile, args->sgeJobName, args->outputFile); fprintf(stderr, "%s\n", cmd); if (system(cmd)) fprintf(stderr, "%s\nFailed to execute qsub command: %s\n", cmd, strerror(errno)), exit(1); // submit the merge sprintf(nam, "%s-merge.sh", args->outputFile); errno = 0; F = fopen(nam, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", nam, strerror(errno)), exit(1); fprintf(F, "#!/bin/sh\n\n"); fprintf(F, ". $SGE_ROOT/$SGE_CELL/common/settings.sh\n"); fprintf(F, "%s -mergebatch -o %s\n", args->execName, args->outputFile); fclose(F); if (args->sgeMergeOpt) sprintf(cmd, "qsub -hold_jid mc%s -cwd -b n -j y -o %s-merge.err %s -N mm%s %s-merge.sh", args->sgeJobName, args->outputFile, args->sgeMergeOpt, args->sgeJobName, args->outputFile); else sprintf(cmd, "qsub -hold_jid mc%s -cwd -b n -j y -o %s-merge.err -N mm%s %s-merge.sh", args->sgeJobName, args->outputFile, args->sgeJobName, args->outputFile); fprintf(stderr, "%s\n", cmd); if (system(cmd)) fprintf(stderr, "%s\nFailed to execute qsub command: %s\n", cmd, strerror(errno)), exit(1); } void prepareBatch(merylArgs *args) { bool fatalError = false; if (args->inputFile == 0L) fprintf(stderr, "ERROR - no input file specified.\n"), fatalError = true; if (args->outputFile == 0L) fprintf(stderr, "ERROR - no output file specified.\n"), fatalError = true; if ((args->doForward == false) && (args->doReverse == false) && (args->doCanonical == false)) fprintf(stderr, "ERROR - need to specify at least one of -f, -r, -C\n"), fatalError = true; if ((args->doForward && args->doReverse) || (args->doForward && args->doCanonical) || (args->doReverse && args->doCanonical)) fprintf(stderr, "ERROR - only one of -f, -r and -C may be specified!\n"), fatalError = true; if (args->lowCount > args->highCount) fprintf(stderr, "ERROR - lowCount > highCount??\n"), fatalError = true; if (args->segmentLimit && args->memoryLimit) fprintf(stderr, "ERROR: Only one of -memory and -segments can be specified.\n"), fatalError=true; if (fatalError) exit(1); // If we were given no segment or memory limit, but threads, we // really want to create n segments. // if ((args->numThreads > 0) && (args->segmentLimit == 0) && (args->memoryLimit == 0)) args->segmentLimit = args->numThreads; { seqStream *seqstr = new seqStream(args->inputFile); args->numBasesActual = 0; for (uint32 i=0; inumberOfSequences(); i++) args->numBasesActual += seqstr->lengthOf(i); merStream *merstr = new merStream(new kMerBuilder(args->merSize), seqstr, true, true); args->numMersActual = merstr->approximateNumberOfMers() + 1; delete merstr; } #warning not submitting prepareBatch to grid #if 0 if ((args->isOnGrid) || (args->sgeJobName == 0L)) { } else { // Shucks, we need to build the merstream file. Lets do it // on the grid! // submitPrepareBatch(args); exit(0); } #endif // If there is a memory limit, figure out how to divide the work into an integer multiple of // numThreads segments. // // Otherwise, if there is a segment limit, split the total number of mers into n pieces. // // Otherwise, we must be doing it all in one fell swoop. // if (args->memoryLimit) { args->mersPerBatch = estimateNumMersInMemorySize(args->merSize, args->memoryLimit, args->numThreads, args->positionsEnabled, args->beVerbose); // Degenerate case; if we can fit more per batch than there are in total, just divide them equally. if (args->mersPerBatch > args->numMersActual) args->mersPerBatch = args->numMersActual / args->numThreads; // Compute how many segments we need, rounding up. args->segmentLimit = (uint64)ceil((double)args->numMersActual / (double)args->mersPerBatch); // Then keep the compute balanced and make the number of segments be a multiple of the number of threads. args->segmentLimit = args->numThreads * (uint32)ceil((double)args->segmentLimit / (double)args->numThreads); } else if (args->segmentLimit) { args->mersPerBatch = (uint64)ceil((double)args->numMersActual / (double)args->segmentLimit); } else { args->mersPerBatch = args->numMersActual; args->segmentLimit = 1; } args->basesPerBatch = (uint64)ceil((double)args->numBasesActual / (double)args->segmentLimit); // Choose the optimal number of buckets to reduce memory usage. Yes, this is already done in // estimateNumMersInMemorySize() (but not saved) and we need to do it for the other cases anyway. // // We use the number of mers per batch + 1 because we need to store the first position after the // last mer. That is, if there are two mers, we will store that the first mer is at position 0, // the second mer is at position 1, and the end of the second mer is at position 2. // args->bucketPointerWidth = logBaseTwo64(args->basesPerBatch + 1); args->numBuckets_log2 = optimalNumberOfBuckets(args->merSize, args->basesPerBatch, args->positionsEnabled); args->numBuckets = (uint64ONE << args->numBuckets_log2); args->merDataWidth = args->merSize * 2 - args->numBuckets_log2; if (args->merDataWidth > SORTED_LIST_WIDTH * 64) { fprintf(stderr, " numMersActual = "F_U64"\n", args->numMersActual); fprintf(stderr, " mersPerBatch = "F_U64"\n", args->mersPerBatch); fprintf(stderr, " basesPerBatch = "F_U64"\n", args->basesPerBatch); fprintf(stderr, " numBuckets = "F_U64" ("F_U32" bits)\n", args->numBuckets, args->numBuckets_log2); fprintf(stderr, " bucketPointerWidth = "F_U32"\n", args->bucketPointerWidth); fprintf(stderr, " merDataWidth = "F_U32"\n", args->merDataWidth); fprintf(stderr, "Sorry! merSize too big! Increase KMER_WORDS in libbio.kmer.H\n"); exit(1); } if (args->beVerbose) { fprintf(stderr, "Computing "F_U64" segments using "F_U32" threads and "F_U64"MB memory ("F_U64"MB if in one batch).\n", args->segmentLimit, args->numThreads, estimateMemory(args->merSize, args->mersPerBatch, args->positionsEnabled) * args->numThreads, estimateMemory(args->merSize, args->numMersActual, args->positionsEnabled)); fprintf(stderr, " numMersActual = "F_U64"\n", args->numMersActual); fprintf(stderr, " mersPerBatch = "F_U64"\n", args->mersPerBatch); fprintf(stderr, " basesPerBatch = "F_U64"\n", args->basesPerBatch); fprintf(stderr, " numBuckets = "F_U64" ("F_U32" bits)\n", args->numBuckets, args->numBuckets_log2); fprintf(stderr, " bucketPointerWidth = "F_U32"\n", args->bucketPointerWidth); fprintf(stderr, " merDataWidth = "F_U32"\n", args->merDataWidth); } } void runSegment(merylArgs *args, uint64 segment) { merStream *M = 0L; merylStreamWriter *W = 0L; speedCounter *C = 0L; uint32 *bucketSizes = 0L; uint64 *bucketPointers = 0L; uint64 *merDataArray[SORTED_LIST_WIDTH] = { 0L }; uint32 *merPosnArray = 0L; // If this segment exists already, skip it. // // XXX: This should be a command line option. // XXX: This should check that the files are complete meryl files. // char *filename = new char [strlen(args->outputFile) + 17]; sprintf(filename, "%s.batch"F_U64".mcdat", args->outputFile, segment); if (AS_UTL_fileExists(filename)) { if (args->beVerbose) fprintf(stderr, "Found result for batch "F_U64" in %s.\n", segment, filename); delete [] filename; return; } if ((args->beVerbose) && (args->segmentLimit > 1)) fprintf(stderr, "Computing segment "F_U64" of "F_U64".\n", segment+1, args->segmentLimit); delete [] filename; // Allocate space for bucket pointers and (temporary) bucket sizes. if (args->beVerbose) fprintf(stderr, " Allocating "F_U64"MB for bucket pointer table ("F_U32" bits wide).\n", (args->numBuckets * args->bucketPointerWidth + 128) >> 23, args->bucketPointerWidth); bucketPointers = new uint64 [(args->numBuckets * args->bucketPointerWidth + 128) >> 6]; if (args->beVerbose) fprintf(stderr, " Allocating "F_U64"MB for counting the size of each bucket.\n", args->numBuckets >> 18); bucketSizes = new uint32 [ args->numBuckets ]; for (uint64 i=args->numBuckets; i--; ) bucketSizes[i] = uint32ZERO; // Position the mer stream at the start of this segments' mers. // The last segment goes until the stream runs out of mers, // everybody else does args->basesPerBatch mers. C = new speedCounter(" Counting mers in buckets: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose); M = new merStream(new kMerBuilder(args->merSize, args->merComp), new seqStream(args->inputFile), true, true); M->setBaseRange(args->basesPerBatch * segment, args->basesPerBatch * segment + args->basesPerBatch); char mstring[256]; if (args->doForward) { while (M->nextMer()) { //fprintf(stderr, "FMER %s\n", M->theFMer().merToString(mstring)); bucketSizes[ args->hash(M->theFMer()) ]++; C->tick(); } } if (args->doReverse) { while (M->nextMer()) { //fprintf(stderr, "RMER %s\n", M->theRMer().merToString(mstring)); bucketSizes[ args->hash(M->theRMer()) ]++; C->tick(); } } if (args->doCanonical) { while (M->nextMer()) { if (M->theFMer() <= M->theRMer()) { //fprintf(stderr, "FMER %s\n", M->theFMer().merToString(mstring)); bucketSizes[ args->hash(M->theFMer()) ]++; } else { //fprintf(stderr, "RMER %s\n", M->theRMer().merToString(mstring)); bucketSizes[ args->hash(M->theRMer()) ]++; } C->tick(); } } delete C; delete M; // Create the hash index using the counts. The hash points // to the end of the bucket; when we add a word, we move the // hash bucket pointer down one. // // When done, we can deallocate the counting table. // if (args->beVerbose) fprintf(stderr, " Creating bucket pointers.\n"); { uint64 mi=0; uint64 mj=0; uint64 mc=0; while (mi < args->numBuckets) { mc += bucketSizes[mi++]; setDecodedValue(bucketPointers, mj, args->bucketPointerWidth, mc); mj += args->bucketPointerWidth; } // Add the location of the end of the table. This is not // modified when adding words, but is used to determine // the size of the last bucket. // setDecodedValue(bucketPointers, mj, args->bucketPointerWidth, mc); } // All done with the counting table, get rid of it. if (args->beVerbose) fprintf(stderr, " Releasing "F_U64"MB from counting the size of each bucket.\n", args->numBuckets >> 18); delete [] bucketSizes; // Allocate space for mer storage and (optional) position data. If mers are bigger than 32, we // allocate full words. if (args->beVerbose) fprintf(stderr, " Allocating "F_U64"MB for mer storage ("F_U32" bits wide).\n", (args->basesPerBatch * args->merDataWidth + 64) >> 23, args->merDataWidth); for (uint64 mword=0, width=args->merDataWidth; width > 0; ) { if (width >= 64) { merDataArray[mword] = new uint64 [ args->basesPerBatch + 1 ]; width -= 64; mword++; } else { merDataArray[mword] = new uint64 [ (args->basesPerBatch * width + 64) >> 6 ]; width = 0; } } // Position data. if (args->positionsEnabled) { if (args->beVerbose) fprintf(stderr, " Allocating "F_U64"MB for mer position storage.\n", (args->basesPerBatch * 32 + 32) >> 23); merPosnArray = new uint32 [ args->basesPerBatch + 1 ]; } C = new speedCounter(" Filling mers into list: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose); M = new merStream(new kMerBuilder(args->merSize, args->merComp), new seqStream(args->inputFile), true, true); M->setBaseRange(args->basesPerBatch * segment, args->basesPerBatch * segment + args->basesPerBatch); while (M->nextMer()) { kMer const &m = ((args->doReverse) || (args->doCanonical && (M->theFMer() > M->theRMer()))) ? M->theRMer() : M->theFMer(); uint64 element = preDecrementDecodedValue(bucketPointers, args->hash(m) * args->bucketPointerWidth, args->bucketPointerWidth); #if SORTED_LIST_WIDTH == 1 // Even though this would work in the general loop below, we // special case one word mers to avoid the loop overhead. // setDecodedValue(merDataArray[0], element * args->merDataWidth, args->merDataWidth, m.endOfMer(args->merDataWidth)); #else for (uint64 mword=0, width=args->merDataWidth; width>0; ) { if (width >= 64) { merDataArray[mword][element] = m.getWord(mword); width -= 64; mword++; } else { setDecodedValue(merDataArray[mword], element * width, width, m.getWord(mword) & uint64MASK(width)); width = 0; } } #endif if (args->positionsEnabled) merPosnArray[element] = M->thePositionInStream(); C->tick(); } delete C; delete M; char *batchOutputFile = new char [strlen(args->outputFile) + 33]; sprintf(batchOutputFile, "%s.batch"F_U64, args->outputFile, segment); C = new speedCounter(" Writing output: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose); W = new merylStreamWriter((args->segmentLimit == 1) ? args->outputFile : batchOutputFile, args->merSize, args->merComp, args->numBuckets_log2, args->positionsEnabled); // Sort each bucket into sortedList, then output the mers // sortedList_t *sortedList = 0L; uint32 sortedListMax = 0; uint32 sortedListLen = 0; for (uint64 bucket=0, bucketPos=0; bucket < args->numBuckets; bucket++) { uint64 st = getDecodedValue(bucketPointers, bucketPos, args->bucketPointerWidth); bucketPos += args->bucketPointerWidth; uint64 ed = getDecodedValue(bucketPointers, bucketPos, args->bucketPointerWidth); if (ed < st) { fprintf(stderr, "ERROR: In segment "F_U64"\n", segment); fprintf(stderr, "ERROR: Bucket "F_U64" (out of "F_U64") ends before it starts!\n", bucket, args->numBuckets); fprintf(stderr, "ERROR: start="F_U64"\n", st); fprintf(stderr, "ERROR: end ="F_U64"\n", ed); } assert(ed >= st); if ((ed - st) > (uint64ONE << 30)) { fprintf(stderr, "ERROR: In segment "F_U64"\n", segment); fprintf(stderr, "ERROR: Bucket "F_U64" (out of "F_U64") is HUGE!\n", bucket, args->numBuckets); fprintf(stderr, "ERROR: start="F_U64"\n", st); fprintf(stderr, "ERROR: end ="F_U64"\n", ed); } // Nothing here? Keep going. if (ed == st) continue; sortedListLen = (uint32)(ed - st); // Allocate more space, if we need to. // if (sortedListLen > sortedListMax) { delete [] sortedList; sortedList = new sortedList_t [2 * sortedListLen + 1]; sortedListMax = 2 * sortedListLen; } // Clear out the sortedList -- if we don't, we leave the high // bits unset which will probably make the sort random. // bzero(sortedList, sizeof(sortedList_t) * sortedListLen); // Unpack the mers into the sorting array // if (args->positionsEnabled) for (uint64 i=st; imerDataWidth; imerDataWidth) sortedList[i-st]._w = getDecodedValue(merDataArray[0], J, args->merDataWidth); #else for (uint64 i=st; imerDataWidth; width>0; ) { if (width >= 64) { sortedList[i-st]._w[mword] = merDataArray[mword][i]; width -= 64; mword++; } else { sortedList[i-st]._w[mword] = getDecodedValue(merDataArray[mword], i * width, width); width = 0; } } } #endif // Sort if there is more than one item // if (sortedListLen > 1) { for (int64 t=(sortedListLen-2)/2; t>=0; t--) adjustHeap(sortedList, t, sortedListLen); for (int64 t=sortedListLen-1; t>0; t--) { sortedList_t tv = sortedList[t]; sortedList[t] = sortedList[0]; sortedList[0] = tv; adjustHeap(sortedList, 0, t); } } // Dump the list of mers to the file. // kMer mer(args->merSize); for (uint32 t=0; ttick(); // Build the complete mer // #if SORTED_LIST_WIDTH == 1 mer.setWord(0, sortedList[t]._w); #else for (uint64 mword=0; mword < SORTED_LIST_WIDTH; mword++) mer.setWord(mword, sortedList[t]._w[mword]); #endif mer.setBits(args->merDataWidth, args->numBuckets_log2, bucket); // Add it if (args->positionsEnabled) W->addMer(mer, 1, &sortedList[t]._p); else W->addMer(mer, 1, 0L); } } delete [] sortedList; delete C; delete W; delete [] batchOutputFile; for (uint32 x=0; xbeVerbose) fprintf(stderr, "Segment "F_U64" finished.\n", segment); } void build(merylArgs *args) { if (!args->countBatch && !args->mergeBatch) prepareBatch(args); // Three choices: // // threaded -- start threads, launch pieces in each thread. This // thread waits for completion and then merges the results. // // batched -- write info file and exit. Compute and merge is done // on separate invocations. // // segmented -- write info file, then do each piece sequentially. // After all pieces finished, do a merge. // // bool doMerge = false; if (args->configBatch) { // Write out our configuration and exit if we are -configbatch // args->writeConfig(); if (args->sgeJobName) { fprintf(stdout, "Batch prepared. Submitting to the grid.\n"); submitCountBatches(args); } else { fprintf(stdout, "Batch prepared. Please run:\n"); for (uint64 s=0; ssegmentLimit; s++) fprintf(stdout, "%s -countbatch "F_U64" -o %s\n", args->execName, s, args->outputFile); fprintf(stdout, "%s -mergebatch -o %s\n", args->execName, args->outputFile); } } else if (args->countBatch) { // Read back the configuration, run the segment and exit if we // are -countbatch // merylArgs *savedArgs = new merylArgs(args->outputFile); savedArgs->beVerbose = args->beVerbose; runSegment(savedArgs, args->batchNumber); delete savedArgs; } else if (args->mergeBatch) { // Check that all the files exist if we are -mergebatch and // continue with execution // // MEMORY LEAK! We should delete this at the end of the // function, but it's a pain, and who cares? // merylArgs *savedArgs = new merylArgs(args->outputFile); savedArgs->beVerbose = args->beVerbose; args = savedArgs; doMerge = true; } else { if (args->numThreads > 1) // Run, using threads. There is a lot of baloney needed, so it's // all in a separate function. // runThreaded(args); else // No special options given, do all the work here and now // for (uint64 s=0; ssegmentLimit; s++) runSegment(args, s); // Either case, we want to merge now. // doMerge = true; } // If there is more than one segment, merge them to get the output. // // We do this by contructing a meryl command line and recursively // (effectively) calling meryl. // // The command line is // // ./meryl -M merge [-v] -s batch1 -s batch2 ... -s batchN -o outputFile // if ((doMerge) && (args->segmentLimit > 1)) { if (args->beVerbose) fprintf(stderr, "Merge results.\n"); int argc = 0; char **argv = new char* [7 + 2 * args->segmentLimit]; bool *arga = new bool [7 + 2 * args->segmentLimit]; arga[argc] = false; argv[argc++] = "meryl-build-merge"; arga[argc] = false; argv[argc++] = "-M"; arga[argc] = false; argv[argc++] = "merge"; if (args->beVerbose) { arga[argc] = false; argv[argc++] = "-v"; } for (uint32 i=0; isegmentLimit; i++) { arga[argc] = false; argv[argc++] = "-s"; arga[argc] = true; argv[argc] = new char [strlen(args->outputFile) + 33]; sprintf(argv[argc], "%s.batch"F_U32, args->outputFile, i); argc++; } arga[argc] = false; argv[argc++] = "-o"; arga[argc] = false; argv[argc++] = args->outputFile; merylArgs *addArgs = new merylArgs(argc, argv); multipleOperations(addArgs); // Cleanup the memory leak. // delete addArgs; for (int i=0; ioutputFile) + 17]; for (uint32 i=0; isegmentLimit; i++) { sprintf(filename, "%s.batch"F_U32".mcidx", args->outputFile, i); unlink(filename); sprintf(filename, "%s.batch"F_U32".mcdat", args->outputFile, i); unlink(filename); sprintf(filename, "%s.batch"F_U32".mcpos", args->outputFile, i); unlink(filename); } delete [] filename; } // If we just merged, delete the merstream file // if (doMerge) { char *filename = new char [strlen(args->outputFile) + 17]; sprintf(filename, "%s.merStream", args->outputFile); unlink(filename); delete [] filename; } } canu-1.3/src/meryl/meryl-dump.C000066400000000000000000000126431272335422300164320ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/dump.C * * Modifications by: * * Brian P. Walenz from 2003-JAN-02 to 2004-APR-07 * are Copyright 2003-2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-APR-12 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAY-23 to 2014-APR-11 * are Copyright 2005,2007-2008,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-FEB-25 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include "meryl.H" #include "libmeryl.H" #include void dumpThreshold(merylArgs *args) { merylStreamReader *M = new merylStreamReader(args->inputFile); char str[1025]; while (M->nextMer()) { if (M->theCount() >= args->numMersEstimated) fprintf(stdout, ">"F_U64"\n%s\n", M->theCount(), M->theFMer().merToString(str)); } delete M; } void dumpPositions(merylArgs *args) { merylStreamReader *M = new merylStreamReader(args->inputFile); char str[1025]; if (M->hasPositions() == false) { fprintf(stderr, "File '%s' contains no position information.\n", args->inputFile); } else { while (M->nextMer()) { fprintf(stdout, ">"F_U64, M->theCount()); for (uint32 i=0; itheCount(); i++) fprintf(stdout, " "F_U32, M->getPosition(i)); fprintf(stdout, "\n%s\n", M->theFMer().merToString(str)); } } delete M; } void countUnique(merylArgs *args) { merylStreamReader *M = new merylStreamReader(args->inputFile); #warning make this a test #if 0 uint64 numDistinct = 0; uint64 numUnique = 0; uint64 numMers = 0; uint64 c = 0; while (M->nextMer()) { c = M->theCount(); numDistinct++; if (c == 1) numUnique++; numMers += c; } assert(numMers == M->numberOfTotalMers()); assert(numDistinct == M->numberOfDistinctMers()); assert(numUnique == M->numberOfUniqueMers()); fprintf(stderr, "OK\n"); #endif fprintf(stdout, "Found "F_U64" mers.\n", M->numberOfTotalMers()); fprintf(stdout, "Found "F_U64" distinct mers.\n", M->numberOfDistinctMers()); fprintf(stdout, "Found "F_U64" unique mers.\n", M->numberOfUniqueMers()); delete M; } void plotHistogram(merylArgs *args) { uint64 distinct = 0; uint64 total = 0; merylStreamReader *M = new merylStreamReader(args->inputFile); fprintf(stderr, "Found "F_U64" mers.\n", M->numberOfTotalMers()); fprintf(stderr, "Found "F_U64" distinct mers.\n", M->numberOfDistinctMers()); fprintf(stderr, "Found "F_U64" unique mers.\n", M->numberOfUniqueMers()); fprintf(stderr, "Largest mercount is "F_U64".\n", M->histogramMaximumCount()); for (uint32 i=1; ihistogramLength(); i++) { uint64 hist = M->histogram(i); if (hist > 0) { distinct += hist; total += hist * i; fprintf(stdout, F_U32"\t"F_U64"\t%.4f\t%.4f\n", i, hist, distinct / (double)M->numberOfDistinctMers(), total / (double)M->numberOfTotalMers()); } } delete M; } void dumpDistanceBetweenMers(merylArgs *args) { merylStreamReader *M = new merylStreamReader(args->inputFile); // This is now tough because we don't know where the sequences end, // and our positions encode position in the chain. uint32 histMax = 64 * 1024 * 1024; uint64 *hist = new uint64 [histMax]; uint64 histHuge = 0; memset(hist, 0, sizeof(uint64) * histMax); if (M->hasPositions() == false) { fprintf(stderr, "File '%s' contains no position information.\n", args->inputFile); } else { while (M->nextMer()) { std::sort(M->thePositions(), M->thePositions() + M->theCount()); for (uint32 i=1; itheCount(); i++) { uint32 d = M->getPosition(i) - M->getPosition(i-1); if (d < histMax) hist[d]++; else histHuge++; } } uint32 maxd = 0; for (uint32 d=0; d 25) ? 50 : 2 * merSize - 2; // Max width of bucket pointer table. // t - prefix stored in the bucket pointer table; number of entries in the table // N - width of a bucket pointer for (uint64 t=2; t < tMax; t++) { for (uint64 N=1; N<40; N++) { uint64 Nmin = uint64ONE << (N - 1); // Minimum number of mers we want to fit in the table uint64 Nmax = uint64ONE << (N); // Maximum number of mers that can fit in the table uint64 bucketsize = (uint64ONE << t) * N; // Size, in bits, of the pointer table uint64 n = (memLimt - bucketsize) / (2*merSize - t + posPerMer); // Number of mers we can fit into mer data table. if ((memLimt > bucketsize) && // pointer table small enough to fit in memory (n > 0) && // at least some space to store mers (n <= Nmax) && // enough space for the mers in the data table (Nmin <= n) && // ...but not more than enough space (maxN < n)) { // this value of t fits more mers that any other seen so far maxN = n; bestT = t; } } } if (beVerbose) fprintf(stdout, "Can fit "F_U64" mers into table with prefix of "F_U64" bits, using %.3fMB (%.3fMB for positions)\n", maxN * numThreads, bestT, (((uint64ONE << bestT) * logBaseTwo64(maxN) + maxN * (2*merSize - bestT + posPerMer)) >> 3) * numThreads / 1048576.0, ((maxN * posPerMer) >> 3) * numThreads / 1048576.0); return(maxN); } uint64 estimateMemory(uint32 merSize, uint64 numMers, bool positionsEnabled) { uint64 posPerMer = (positionsEnabled == false) ? 0 : 32; uint64 tMax = (merSize > 25) ? 50 : 2 * merSize - 2; uint64 tMin = tMax; uint64 memMin = UINT64_MAX; for (uint64 t=2; t < tMax; t++) { uint64 N = logBaseTwo64(numMers); // Width of the bucket pointer table uint64 memUsed = ((uint64ONE << t) * logBaseTwo64(numMers) + numMers * (2 * merSize - t + posPerMer)) >> 3; if (memUsed < memMin) { tMin = t; memMin = memUsed; } //fprintf(stderr, "t=%2lu N=%2lu memUsed=%16lu -- tMin=%2lu memMin=%16lu\n", // t, N, memUsed, tMin, memMin); } return(memMin >> 20); } uint32 optimalNumberOfBuckets(uint32 merSize, uint64 numMers, bool positionsEnabled) { uint64 opth = ~uint64ZERO; uint64 opts = ~uint64ZERO; uint64 h = 0; uint64 s = 0; uint64 hwidth = logBaseTwo64(numMers); // Positions consume space too, but only if enabled. Probably // doesn't matter here. // uint64 posPerMer = 0; if (positionsEnabled) posPerMer = 32; // Find the table size (in bits, h) that minimizes memory usage // for the given merSize and numMers // // We have two tables: // the bucket pointers num buckets * pointer width == 2 << h * hwidth // the mer data: num mers * (mersize - hwidth) // uint64 hmax = 64 - logBaseTwo64(hwidth + numMers * (2 * merSize - h)); for (h=2; h<=hmax && h<2*merSize; h++) { s = (uint64ONE << h) * hwidth + numMers * (2 * merSize - h + posPerMer); //fprintf(stderr, "optimalNumberOfBuckets()-- h="F_U64" s="F_U64"\n", h, s); if (s < opts) { opth = h; opts = s; } } return((uint32)opth); } void estimate(merylArgs *args) { if (args->inputFile) { merStream M(new kMerBuilder(args->merSize, args->merComp), new seqStream(args->inputFile), true, true); speedCounter C(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose); if (args->beVerbose) fprintf(stderr, "Counting mers in '%s'\n", args->inputFile); args->numMersEstimated = 0; while (M.nextMer()) { C.tick(); args->numMersEstimated++; } C.finish(); } uint32 opth = optimalNumberOfBuckets(args->merSize, args->numMersEstimated, args->positionsEnabled); uint64 memu = ((uint64ONE << opth) * logBaseTwo64(args->numMersEstimated+1) + args->numMersEstimated * (2 * args->merSize - opth)); fprintf(stderr, F_U64" "F_U32"-mers can be computed using "F_U64"MB memory.\n", args->numMersEstimated, args->merSize, memu >> 23); } canu-1.3/src/meryl/meryl-merge-listmerge.C000066400000000000000000000315271272335422300205570ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/merge.listmerge.C * * Modifications by: * * Brian P. Walenz from 2008-JUN-20 to 2014-APR-11 * are Copyright 2008,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include "meryl.H" #include "libmeryl.H" using namespace std; #include struct mMer { kMer _mer; uint32 _cnt; uint32 _off; uint32 _nxt; uint32 _stp; }; class mMerList { public: mMerList(uint32 maxSize) { _posLen = 0; _posMax = 2 * maxSize; _pos = new uint32 [_posMax]; _mmmLen = 0; _mmmMax = maxSize; _mmm = new mMer [_mmmMax]; _tip = ~uint32ZERO; _fre = 0; for (uint32 i=0; i<_mmmMax; i++) { _mmm[i]._cnt = 0; _mmm[i]._off = 0; _mmm[i]._nxt = i+1; _mmm[i]._stp = 0; } _mmm[_mmmMax-1]._nxt = ~uint32ZERO; }; ~mMerList() { delete [] _pos; delete [] _mmm; }; bool loadMore(void) { return((_mmmMax < _tip) || (_mmm[_tip]._stp == 1)); }; uint32 length(void) { return(_mmmLen); }; kMer *pop(uint32 &cnt, uint32* &pos) { kMer *ret = 0L; //fprintf(stderr, "POP tip="uint32FMT"\n", _tip); if (_tip < _mmmMax) { uint32 f = _tip; ret = &_mmm[f]._mer; cnt = _mmm[f]._cnt; pos = (_mmm[f]._off != ~uint32ZERO) ? _pos + _mmm[f]._off : 0L; // Move tip to the next thing _tip = _mmm[f]._nxt; // And append this one to the free list. _mmm[f]._nxt = _fre; _fre = f; _mmmLen--; //fprintf(stderr, "POP f="uint32FMT" tip="uint32FMT" len="uint32FMT"\n", f, _tip, _mmmLen); } return(ret); }; // rebuild the position list, squeezes out empty items void rebuild(void) { if (_posLen > 0) { assert(0); uint32 *np = new uint32 [_posMax]; _posLen = 0; for (uint32 i=0; i<_mmmLen; i++) { mMer *m = _mmm + i; if (m->_off != ~uint32ZERO) { _mmm[_mmmLen]._off = _posLen; for (uint32 p=0; p_cnt; p++, _posLen++) np[_posLen] = _pos[p]; } } delete [] _pos; _pos = np; } }; // Read more mers from the file void read(merylStreamReader *R, uint32 num, bool loadAll) { uint32 xxx = 0; uint32 las = ~uint32ZERO; uint32 pos = _tip; bool stop = false; //fprintf(stderr, "read()- loading "uint32FMT"\n", num); assert(_mmmLen + num < _mmmMax); // Load until we hit the sentinal. if (loadAll == false) num = ~uint32ZERO; for (xxx=0; (xxx < num) && (stop == false) && (R->nextMer()); xxx++) { // Insert into a free node uint32 fre = _fre; _fre = _mmm[fre]._nxt; _mmm[fre]._mer = R->theFMer(); _mmm[fre]._cnt = R->theCount(); _mmm[fre]._off = ~uint32ZERO; _mmm[fre]._stp = 0; uint32 *ppp = R->thePositions(); if (ppp) { _mmm[fre]._off = _posLen; if (_posMax <= _posLen + _mmm[fre]._cnt) { fprintf(stderr, "Reallocate _pos\n"); _posMax *= 2; uint32 *tmp = new uint32 [_posMax]; memcpy(tmp, _pos, sizeof(uint32) * _posLen); delete [] _pos; _pos = tmp; } for (uint32 i=0; i<_mmm[fre]._cnt; i++, _posLen++) _pos[_posLen] = ppp[i]; } // Keep count _mmmLen++; // Figure out where to put it in the list. New duplicates must // go AFTER the existing -- that's the job of <=. while ((pos < _mmmMax) && (_mmm[pos]._mer <= R->theFMer())) { las = pos; pos = _mmm[pos]._nxt; } if (_mmmMax < _tip) { // No tip, make new list. _mmm[fre]._nxt = _tip; _tip = fre; las = ~uint32ZERO; pos = _tip; } else if (_mmmMax < las) { // Valid list, but we want to insert before the start _mmm[fre]._nxt = _tip; _tip = fre; las = ~uint32ZERO; pos = _tip; } else if (pos < _mmmMax) { // Valid pos, insert in the middle (after las, before pos) _mmm[fre]._nxt = _mmm[las]._nxt; _mmm[las]._nxt = fre; las = fre; //pos = _mmm[las]._nxt; } else { // Have a list, but we ran off the end, append (after las) _mmm[fre]._nxt = ~uint32ZERO; _mmm[las]._nxt = fre; pos = fre; if (loadAll == false) stop = true; } } // Set the sentinal. This forces us to load more mers. // if (loadAll == true) { //fprintf(stderr, "read()-- stop on tip = "uint32FMT"\n", las); _mmm[las]._stp = 1; } //fprintf(stderr, "read()-- now up to "uint32FMT" mers ("uint32FMT" pos); loaded "uint32FMT" out of "uint32FMT" requested.\n", _mmmLen, _posLen, xxx, num); }; private: uint32 _posLen; uint32 _posMax; uint32 *_pos; uint32 _mmmLen; uint32 _mmmMax; mMer *_mmm; uint32 _tip; uint32 _fre; }; void multipleOperations(merylArgs *args) { if (args->mergeFilesLen < 2) { fprintf(stderr, "ERROR - must have at least two databases (you gave "uint32FMT")!\n", args->mergeFilesLen); exit(1); } if (args->outputFile == 0L) { fprintf(stderr, "ERROR - no output file specified.\n"); exit(1); } if ((args->personality != PERSONALITY_MERGE) && (args->personality != PERSONALITY_MIN) && (args->personality != PERSONALITY_MINEXIST) && (args->personality != PERSONALITY_MAX) && (args->personality != PERSONALITY_ADD) && (args->personality != PERSONALITY_AND) && (args->personality != PERSONALITY_NAND) && (args->personality != PERSONALITY_OR) && (args->personality != PERSONALITY_XOR)) { fprintf(stderr, "ERROR - only personalities min, minexist, max, add, and, nand, or, xor\n"); fprintf(stderr, "ERROR - are supported in multipleOperations().\n"); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); } uint32 maxSize = 64 * 1024 * 1024; merylStreamReader **R = new merylStreamReader* [args->mergeFilesLen]; merylStreamWriter *W = 0L; mMerList *M = new mMerList(maxSize + maxSize / 4); for (uint32 i=0; imergeFilesLen; i++) R[i] = new merylStreamReader(args->mergeFiles[i]); // Verify that the mersizes are all the same // bool fail = false; uint32 merSize = R[0]->merSize(); uint32 merComp = R[0]->merCompression(); for (uint32 i=0; imergeFilesLen; i++) { fail |= (merSize != R[i]->merSize()); fail |= (merComp != R[i]->merCompression()); } if (fail) fprintf(stderr, "ERROR: mer size or compression level differ.\n"), exit(1); // Open the output file, using the largest prefix size found in the // input/mask files. // uint32 prefixSize = 0; for (uint32 i=0; imergeFilesLen; i++) if (prefixSize < R[i]->prefixSize()) prefixSize = R[i]->prefixSize(); W = new merylStreamWriter(args->outputFile, merSize, merComp, prefixSize); // Load mers from all files, remember the largest mer we load. // bool loadAll = true; for (uint32 i=0; imergeFilesLen; i++) { M->read(R[i], maxSize / args->mergeFilesLen, loadAll); loadAll = false; } fprintf(stderr, "Initial load: length="uint32FMT"\n", M->length()); bool moreStuff = true; kMer currentMer; // The current mer we're operating on uint32 currentCount = uint32ZERO; // The count (operation dependent) of this mer uint32 currentTimes = uint32ZERO; // Number of files it's in uint32 currentPositionsMax = 0; uint32 *currentPositions = 0L; kMer *thisMer; // The mer we just read uint32 thisCount = uint32ZERO; // The count of the mer we just read uint32 *thisPositions = 0L; speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose); currentMer.setMerSize(merSize); while (moreStuff) { // Load more stuff if needed. // if (M->loadMore() == true) { M->rebuild(); uint32 additionalLoading = 8192; if (maxSize / args->mergeFilesLen > M->length()) additionalLoading = maxSize / args->mergeFilesLen - M->length(); loadAll = true; for (uint32 i=0; imergeFilesLen; i++) { if (R[i]->validMer()) { M->read(R[i], additionalLoading, loadAll); loadAll = false; } } } // All done? Exit. if (M->length() == 0) moreStuff = false; thisMer = M->pop(thisCount, thisPositions); // If we've hit a different mer, write out the last one if ((M->length() == 0) || (*thisMer != currentMer)) { switch (args->personality) { case PERSONALITY_MIN: if (currentTimes == args->mergeFilesLen) W->addMer(currentMer, currentCount); break; case PERSONALITY_MERGE: case PERSONALITY_MINEXIST: case PERSONALITY_MAX: case PERSONALITY_ADD: W->addMer(currentMer, currentCount, currentPositions); break; case PERSONALITY_AND: if (currentTimes == args->mergeFilesLen) W->addMer(currentMer, currentCount); break; case PERSONALITY_NAND: if (currentTimes != args->mergeFilesLen) W->addMer(currentMer, currentCount); break; case PERSONALITY_OR: W->addMer(currentMer, currentCount); break; case PERSONALITY_XOR: if ((currentTimes % 2) == 1) W->addMer(currentMer, currentCount); break; default: fprintf(stderr, "ERROR - invalid personality in multipleOperations::write\n"); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); break; } currentMer = *thisMer; currentCount = uint32ZERO; currentTimes = uint32ZERO; C->tick(); } if (moreStuff == false) break; // Perform the operation switch (args->personality) { case PERSONALITY_MERGE: if (thisPositions) { if (currentPositionsMax == 0) { currentPositionsMax = 1048576; currentPositions = new uint32 [currentPositionsMax]; } if (currentPositionsMax < currentCount + thisCount) { while (currentPositionsMax < currentCount + thisCount) currentPositionsMax *= 2; uint32 *t = new uint32 [currentPositionsMax]; memcpy(t, currentPositions, sizeof(uint32) * currentCount); delete [] currentPositions; currentPositions = t; } if (thisCount < 16) { for (uint32 i=0; i thisCount) currentCount = thisCount; } break; case PERSONALITY_MAX: if (currentCount < thisCount) currentCount = thisCount; break; case PERSONALITY_ADD: currentCount += thisCount; break; case PERSONALITY_AND: case PERSONALITY_NAND: case PERSONALITY_OR: case PERSONALITY_XOR: currentCount = 1; break; default: fprintf(stderr, "ERROR - invalid personality in multipleOperations::operate\n"); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); break; } currentTimes++; } for (uint32 i=0; imergeFilesLen; i++) delete R[i]; delete R; delete W; delete M; delete C; } canu-1.3/src/meryl/meryl-merge-qsort.C000066400000000000000000000333051272335422300177300ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/merge.qsort.C * * Modifications by: * * Brian P. Walenz from 2008-JUN-20 to 2014-APR-11 * are Copyright 2008,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include "meryl.H" #include "libmeryl.H" using namespace std; #include struct mMer { kMer _mer; uint32 _cnt; uint32 _off; }; static int mMerGreaterThan(void const *a, void const *b) { mMer const *A = (mMer const *)a; mMer const *B = (mMer const *)b; return(B->_mer.qsort_less(A->_mer)); } class mMerList { public: mMerList(uint32 maxSize) { _posLen = 0; _posMax = 2 * maxSize; _pos = new uint32 [_posMax]; _mmmLen = 0; _mmmMax = maxSize; _mmm = new mMer [_mmmMax]; }; ~mMerList() { delete [] _pos; delete [] _mmm; }; uint32 length(void) { return(_mmmLen); }; // Until we sort, first() is the last thing loaded. // After we sort, first() is the lowest mer in the set. kMer &first(void) { return(_mmm[_mmmLen-1]._mer); }; //kMer &last(void) { return(_mmm[0]._mer); }; //kMer &get(uint32 i) { return(_mmm[i]._mer); }; // Return the first (sorted order) thing in the list -- it's the last on the list. kMer *pop(uint32 &cnt, uint32* &pos) { if (_mmmLen == 0) return(0L); _mmmLen--; assert(_sorted); cnt = _mmm[_mmmLen]._cnt; pos = 0L; if (_mmm[_mmmLen]._off != ~uint32ZERO) pos = _pos + _mmm[_mmmLen]._off; return(&_mmm[_mmmLen]._mer); } // rebuild the position list, squeezes out empty items void rebuild(void) { if (_posLen > 0) { uint32 *np = new uint32 [_posMax]; _posLen = 0; for (uint32 i=0; i<_mmmLen; i++) { mMer *m = _mmm + i; if (m->_off != ~uint32ZERO) { _mmm[_mmmLen]._off = _posLen; for (uint32 p=0; p_cnt; p++, _posLen++) np[_posLen] = _pos[p]; } } delete [] _pos; _pos = np; } }; // Read more mers from the file void read(merylStreamReader *R, uint32 num) { uint32 xxx = 0; if (_mmmLen + num >= _mmmMax) { fprintf(stderr, "Reallocate _mmm\n"); _mmmMax = _mmmMax + 2 * num; mMer *tmp = new mMer [_mmmMax]; memcpy(tmp, _mmm, sizeof(mMer) * _mmmLen); delete [] _mmm; _mmm = tmp; } _sorted = false; R->nextMer(); for (xxx=0; (xxx < num) && (R->validMer()); xxx++) { if (_mmmMax <= _mmmLen) { fprintf(stderr, "Reallocate _mmm\n"); _mmmMax *= 2; mMer *tmp = new mMer [_mmmMax]; memcpy(tmp, _mmm, sizeof(mMer) * _mmmLen); delete [] _mmm; _mmm = tmp; } _mmm[_mmmLen]._mer = R->theFMer(); _mmm[_mmmLen]._cnt = R->theCount(); _mmm[_mmmLen]._off = ~uint32ZERO; uint32 *pos = R->thePositions(); if (pos) { _mmm[_mmmLen]._off = _posLen; if (_posMax <= _posLen + _mmm[_mmmLen]._cnt) { fprintf(stderr, "Reallocate _pos\n"); _posMax *= 2; uint32 *tmp = new uint32 [_posMax]; memcpy(tmp, _pos, sizeof(uint32) * _posLen); delete [] _pos; _pos = tmp; } for (uint32 i=0; i<_mmm[_mmmLen]._cnt; i++, _posLen++) _pos[_posLen] = pos[i]; } _mmmLen++; R->nextMer(); } //fprintf(stderr, "read()-- now up to "uint32FMT" mers ("uint32FMT" pos); loaded "uint32FMT" out of "uint32FMT" requested.\n", _mmmLen, _posLen, xxx, num); }; // Sort our list of mers void sort(void) { if (_sorted == false) { //fprintf(stderr, "SORT BEG\n"); qsort_mt(_mmm, _mmmLen, sizeof(mMer), mMerGreaterThan, 8, 32 * 1024); _sorted = true; //fprintf(stderr, "SORT END\n"); } }; private: bool _sorted; uint32 _posLen; uint32 _posMax; uint32 *_pos; uint32 _mmmLen; uint32 _mmmMax; mMer *_mmm; }; void multipleOperations(merylArgs *args) { char debugstring[256]; char debugstring2[256]; if (args->mergeFilesLen < 2) { fprintf(stderr, "ERROR - must have at least two databases (you gave "uint32FMT")!\n", args->mergeFilesLen); exit(1); } if (args->outputFile == 0L) { fprintf(stderr, "ERROR - no output file specified.\n"); exit(1); } if ((args->personality != PERSONALITY_MERGE) && (args->personality != PERSONALITY_MIN) && (args->personality != PERSONALITY_MINEXIST) && (args->personality != PERSONALITY_MAX) && (args->personality != PERSONALITY_ADD) && (args->personality != PERSONALITY_AND) && (args->personality != PERSONALITY_NAND) && (args->personality != PERSONALITY_OR) && (args->personality != PERSONALITY_XOR)) { fprintf(stderr, "ERROR - only personalities min, minexist, max, add, and, nand, or, xor\n"); fprintf(stderr, "ERROR - are supported in multipleOperations().\n"); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); } merylStreamReader **R = new merylStreamReader* [args->mergeFilesLen]; merylStreamWriter *W = 0L; uint32 maxSize = 512 * 1024; mMerList *M = new mMerList(maxSize + maxSize / 4); // Open the input files and load some mers - we need to do this // just so we can check the mersizes/compression next. // for (uint32 i=0; imergeFilesLen; i++) { R[i] = new merylStreamReader(args->mergeFiles[i]); M->read(R[i], 1 + i); } // Verify that the mersizes are all the same // bool fail = false; uint32 merSize = R[0]->merSize(); uint32 merComp = R[0]->merCompression(); for (uint32 i=0; imergeFilesLen; i++) { fail |= (merSize != R[i]->merSize()); fail |= (merComp != R[i]->merCompression()); } if (fail) fprintf(stderr, "ERROR: mer sizes (or compression level) differ.\n"), exit(1); // Open the output file, using the largest prefix size found in the // input/mask files. // uint32 prefixSize = 0; for (uint32 i=0; imergeFilesLen; i++) if (prefixSize < R[i]->prefixSize()) prefixSize = R[i]->prefixSize(); W = new merylStreamWriter(args->outputFile, merSize, merComp, prefixSize); kMer lastLoaded; lastLoaded.setMerSize(merSize); lastLoaded.smallest(); // Load mers from all files, remember the largest mer we load. // for (uint32 i=0; imergeFilesLen; i++) { M->read(R[i], maxSize / args->mergeFilesLen); if (lastLoaded < M->first()) lastLoaded = M->first(); } // Make sure all files have at least that largest mer loaded. // for (uint32 i=0; imergeFilesLen; i++) while (R[i]->validMer() && (R[i]->theFMer() <= lastLoaded)) M->read(R[i], 2 * 1024); fprintf(stderr, "Initial load: length="uint32FMT" lastLoaded=%s\n", M->length(), lastLoaded.merToString(debugstring)); M->sort(); bool allLoaded = false; bool moreStuff = true; kMer currentMer; // The current mer we're operating on uint32 currentCount = uint32ZERO; // The count (operation dependent) of this mer uint32 currentTimes = uint32ZERO; // Number of files it's in uint32 currentPositionsMax = 0; uint32 *currentPositions = 0L; kMer *thisMer; // The mer we just read uint32 thisCount = uint32ZERO; // The count of the mer we just read uint32 *thisPositions = 0L; speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose); currentMer.setMerSize(merSize); while (moreStuff) { // Load more stuff if needed. M is sorted, so first() is the // smallest mer in the set - we're good up to and including // lastLoaded. // if ((allLoaded == false) && ((M->length() == 0) || (lastLoaded < M->first()))) { #if 0 if (M->length() > 0) fprintf(stderr, "LOADMORE length="uint32FMT" lastLoaded=%s first=%s\n", M->length(), lastLoaded.merToString(debugstring2), M->first().merToString(debugstring)); else fprintf(stderr, "LOADMORE length="uint32FMT" lastLoaded=%s first=EMPTY\n", M->length(), lastLoaded.merToString(debugstring2)); #endif // We need to copy all the mers currently loaded into fresh // storage, so we can deallocate the position storage. Yucky. // M->rebuild(); allLoaded = true; // Load more stuff to give us a large collection of mers // uint32 additionalLoading = 8192; if (maxSize / args->mergeFilesLen > M->length()) additionalLoading = maxSize / args->mergeFilesLen - M->length(); //fprintf(stderr, "LOADMORE adding "uint32FMT" from each file\n", additionalLoading); lastLoaded.setMerSize(merSize); lastLoaded.smallest(); for (uint32 i=0; imergeFilesLen; i++) { if (R[i]->validMer()) { M->read(R[i], additionalLoading); if (lastLoaded < M->first()) lastLoaded = M->first(); allLoaded = false; } } // Make sure all files have at least that largest mer loaded. // for (uint32 i=0; imergeFilesLen; i++) while (R[i]->validMer() && (R[i]->theFMer() <= lastLoaded)) M->read(R[i], 2 * 1024); M->sort(); } // All done? Exit. if (M->length() == 0) moreStuff = false; thisMer = M->pop(thisCount, thisPositions); // If we've hit a different mer, write out the last one if ((M->length() == 0) || (*thisMer != currentMer)) { switch (args->personality) { case PERSONALITY_MIN: if (currentTimes == args->mergeFilesLen) W->addMer(currentMer, currentCount); break; case PERSONALITY_MERGE: case PERSONALITY_MINEXIST: case PERSONALITY_MAX: case PERSONALITY_ADD: W->addMer(currentMer, currentCount, currentPositions); break; case PERSONALITY_AND: if (currentTimes == args->mergeFilesLen) W->addMer(currentMer, currentCount); break; case PERSONALITY_NAND: if (currentTimes != args->mergeFilesLen) W->addMer(currentMer, currentCount); break; case PERSONALITY_OR: W->addMer(currentMer, currentCount); break; case PERSONALITY_XOR: if ((currentTimes % 2) == 1) W->addMer(currentMer, currentCount); break; default: fprintf(stderr, "ERROR - invalid personality in multipleOperations::write\n"); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); break; } currentMer = *thisMer; currentCount = uint32ZERO; currentTimes = uint32ZERO; C->tick(); } if (moreStuff == false) break; // Perform the operation switch (args->personality) { case PERSONALITY_MERGE: if (thisPositions) { if (currentPositionsMax == 0) { currentPositionsMax = 1048576; currentPositions = new uint32 [currentPositionsMax]; } if (currentPositionsMax < currentCount + thisCount) { while (currentPositionsMax < currentCount + thisCount) currentPositionsMax *= 2; uint32 *t = new uint32 [currentPositionsMax]; memcpy(t, currentPositions, sizeof(uint32) * currentCount); delete [] currentPositions; currentPositions = t; } if (thisCount < 16) { for (uint32 i=0; i thisCount) currentCount = thisCount; } break; case PERSONALITY_MAX: if (currentCount < thisCount) currentCount = thisCount; break; case PERSONALITY_ADD: currentCount += thisCount; break; case PERSONALITY_AND: case PERSONALITY_NAND: case PERSONALITY_OR: case PERSONALITY_XOR: currentCount = 1; break; default: fprintf(stderr, "ERROR - invalid personality in multipleOperations::operate\n"); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); break; } currentTimes++; } for (uint32 i=0; imergeFilesLen; i++) delete R[i]; delete R; delete W; delete M; delete C; } canu-1.3/src/meryl/meryl-merge.C000066400000000000000000000216551272335422300165670ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/merge.C * * Modifications by: * * Brian P. Walenz from 2003-JAN-02 to 2004-APR-08 * are Copyright 2003-2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-APR-09 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAY-23 to 2014-APR-11 * are Copyright 2005-2009,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "meryl.H" #include "libmeryl.H" void multipleOperations(merylArgs *args) { if (args->mergeFilesLen < 2) { fprintf(stderr, "ERROR - must have at least two databases (you gave "F_U32")!\n", args->mergeFilesLen); exit(1); } if (args->outputFile == 0L) { fprintf(stderr, "ERROR - no output file specified.\n"); exit(1); } if ((args->personality != PERSONALITY_MERGE) && (args->personality != PERSONALITY_MIN) && (args->personality != PERSONALITY_MINEXIST) && (args->personality != PERSONALITY_MAX) && (args->personality != PERSONALITY_MAXEXIST) && (args->personality != PERSONALITY_ADD) && (args->personality != PERSONALITY_AND) && (args->personality != PERSONALITY_NAND) && (args->personality != PERSONALITY_OR) && (args->personality != PERSONALITY_XOR)) { fprintf(stderr, "ERROR - only personalities min, minexist, max, maxexist, add, and, nand, or, xor\n"); fprintf(stderr, "ERROR - are supported in multipleOperations(). (%d)\n", args->personality); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); } merylStreamReader **R = new merylStreamReader* [args->mergeFilesLen]; merylStreamWriter *W = 0L; // Open the input files, read in the first mer // for (uint32 i=0; imergeFilesLen; i++) { R[i] = new merylStreamReader(args->mergeFiles[i]); R[i]->nextMer(); } // Verify that the mersizes are all the same // bool fail = false; uint32 merSize = R[0]->merSize(); uint32 merComp = R[0]->merCompression(); for (uint32 i=0; imergeFilesLen; i++) { fail |= (merSize != R[i]->merSize()); fail |= (merComp != R[i]->merCompression()); } if (fail) fprintf(stderr, "ERROR: mer sizes (or compression level) differ.\n"), exit(1); // Open the output file, using the largest prefix size found in the // input/mask files. // uint32 prefixSize = 0; for (uint32 i=0; imergeFilesLen; i++) if (prefixSize < R[i]->prefixSize()) prefixSize = R[i]->prefixSize(); W = new merylStreamWriter(args->outputFile, merSize, merComp, prefixSize, args->positionsEnabled); // We will find the smallest mer in any file, and count the number of times // it is present in the input files. bool moreInput = true; kMer currentMer; // The current mer we're operating on uint32 currentCount = uint32ZERO; // The count (operation dependent) of this mer uint32 currentTimes = uint32ZERO; // Number of files it's in uint32 currentPositionsMax = 0; uint32 *currentPositions = 0L; kMer thisMer; // The mer we just read uint32 thisFile = ~uint32ZERO; // The file we read it from uint32 thisCount = uint32ZERO; // The count of the mer we just read speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, args->beVerbose); currentMer.setMerSize(merSize); thisMer.setMerSize(merSize); while (moreInput) { // Find the smallest mer present in any input file. // moreInput = false; thisMer.clear(); thisFile = ~uint32ZERO; thisCount = uint32ZERO; // Load thisMer with the first valid mer for (uint32 i=0; imergeFilesLen && !moreInput; i++) if (R[i]->validMer()) { moreInput = true; thisCount = R[i]->theCount(); thisFile = i; thisMer = R[i]->theFMer(); } // Now find the smallest one if (moreInput) { for (uint32 i=thisFile+1; imergeFilesLen; i++) if ((R[i]->validMer()) && (R[i]->theFMer()) < thisMer) { moreInput = true; thisCount = R[i]->theCount(); thisFile = i; thisMer = R[i]->theFMer(); } } // If we've hit a different mer, write out the last one if ((moreInput == false) || (thisMer != currentMer)) { switch (args->personality) { case PERSONALITY_MIN: case PERSONALITY_MAX: if (currentTimes == args->mergeFilesLen) W->addMer(currentMer, currentCount); break; case PERSONALITY_MERGE: case PERSONALITY_MINEXIST: case PERSONALITY_MAXEXIST: case PERSONALITY_ADD: W->addMer(currentMer, currentCount, currentPositions); break; case PERSONALITY_AND: if (currentTimes == args->mergeFilesLen) W->addMer(currentMer, currentCount); break; case PERSONALITY_NAND: if (currentTimes != args->mergeFilesLen) W->addMer(currentMer, currentCount); break; case PERSONALITY_OR: W->addMer(currentMer, currentCount); break; case PERSONALITY_XOR: if ((currentTimes % 2) == 1) W->addMer(currentMer, currentCount); break; default: fprintf(stderr, "ERROR - invalid personality in multipleOperations::write\n"); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); break; } currentMer = thisMer; currentCount = uint32ZERO; currentTimes = uint32ZERO; C->tick(); } // All done? Exit. if (moreInput == false) continue; // Perform the operation switch (args->personality) { case PERSONALITY_MERGE: if (R[thisFile]->thePositions()) { if (currentPositionsMax == 0) { currentPositionsMax = 1048576; currentPositions = new uint32 [currentPositionsMax]; } if (currentPositionsMax < currentCount + thisCount) { while (currentPositionsMax < currentCount + thisCount) currentPositionsMax *= 2; uint32 *t = new uint32 [currentPositionsMax]; memcpy(t, currentPositions, sizeof(uint32) * currentCount); delete [] currentPositions; currentPositions = t; } if (thisCount < 16) { uint32 *p = R[thisFile]->thePositions(); for (uint32 i=0; ithePositions(), sizeof(uint32) * thisCount); } } // Otherwise, we're the same as ADD. currentCount += thisCount; break; case PERSONALITY_MIN: case PERSONALITY_MINEXIST: if (currentTimes == 0) { currentCount = thisCount; } else { if (currentCount > thisCount) currentCount = thisCount; } break; case PERSONALITY_MAX: case PERSONALITY_MAXEXIST: if (currentCount < thisCount) currentCount = thisCount; break; case PERSONALITY_ADD: currentCount += thisCount; break; case PERSONALITY_AND: case PERSONALITY_NAND: case PERSONALITY_OR: case PERSONALITY_XOR: currentCount = 1; break; default: fprintf(stderr, "ERROR - invalid personality in multipleOperations::operate\n"); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); break; } currentTimes++; // Move the file we just read from to the next mer R[thisFile]->nextMer(); } for (uint32 i=0; imergeFilesLen; i++) delete R[i]; delete R; delete W; delete C; } canu-1.3/src/meryl/meryl-unaryOp.C000066400000000000000000000063511272335422300171210ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/unaryOp.C * * Modifications by: * * Brian P. Walenz from 2003-JAN-02 to 2004-APR-07 * are Copyright 2003-2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2007-OCT-12 to 2008-JUN-09 * are Copyright 2007-2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-05 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include "meryl.H" #include "libmeryl.H" void unaryOperations(merylArgs *args) { if (args->mergeFilesLen != 1) { fprintf(stderr, "ERROR - must have exactly one file!\n"); exit(1); } if (args->outputFile == 0L) { fprintf(stderr, "ERROR - no output file specified.\n"); exit(1); } if ((args->personality != PERSONALITY_LEQ) && (args->personality != PERSONALITY_GEQ) && (args->personality != PERSONALITY_EQ)) { fprintf(stderr, "ERROR - only personalities lessthan, lessthanorequal,\n"); fprintf(stderr, "ERROR - greaterthan, greaterthanorequal, and equal\n"); fprintf(stderr, "ERROR - are supported in unaryOperations().\n"); fprintf(stderr, "ERROR - this is a coding error, not a user error.\n"); exit(1); } // Open the input and output files -- we don't know the number // unique, distinct, and total until after the operation, so we // leave them zero. // merylStreamReader *R = new merylStreamReader(args->mergeFiles[0]); merylStreamWriter *W = new merylStreamWriter(args->outputFile, R->merSize(), R->merCompression(), R->prefixSize(), R->hasPositions()); switch (args->personality) { case PERSONALITY_LEQ: while (R->nextMer()) if (R->theCount() <= args->desiredCount) W->addMer(R->theFMer(), R->theCount(), R->thePositions()); break; case PERSONALITY_GEQ: while (R->nextMer()) if (R->theCount() >= args->desiredCount) W->addMer(R->theFMer(), R->theCount(), R->thePositions()); break; case PERSONALITY_EQ: while (R->nextMer()) if (R->theCount() == args->desiredCount) W->addMer(R->theFMer(), R->theCount(), R->thePositions()); break; } delete R; delete W; } canu-1.3/src/meryl/meryl.C000066400000000000000000000054341272335422300154670ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/meryl.C * * Modifications by: * * Brian P. Walenz from 2003-JAN-02 to 2004-APR-07 * are Copyright 2003-2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-MAR-25 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAY-23 to 2009-AUG-07 * are Copyright 2005,2007-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2014-DEC-08 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include #include "meryl.H" int main(int argc, char **argv) { merylArgs *args = new merylArgs(argc, argv); switch (args->personality) { case 'P': estimate(args); break; case 'B': build(args); break; case 'd': dumpDistanceBetweenMers(args); break; case 't': dumpThreshold(args); break; case 'p': dumpPositions(args); break; case 'c': countUnique(args); break; case 'h': plotHistogram(args); break; case PERSONALITY_MIN: case PERSONALITY_MINEXIST: case PERSONALITY_MAX: case PERSONALITY_MAXEXIST: case PERSONALITY_ADD: case PERSONALITY_AND: case PERSONALITY_NAND: case PERSONALITY_OR: case PERSONALITY_XOR: multipleOperations(args); break; case PERSONALITY_SUB: case PERSONALITY_ABS: case PERSONALITY_DIVIDE: binaryOperations(args); break; case PERSONALITY_LEQ: case PERSONALITY_GEQ: case PERSONALITY_EQ: unaryOperations(args); break; default: args->usage(); fprintf(stderr, "%s: unknown personality. Specify -P, -B, -S or -M!\n", args->execName); exit(1); break; } delete args; return(0); } canu-1.3/src/meryl/meryl.H000066400000000000000000000123011272335422300154630ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/meryl.H * * Modifications by: * * Brian P. Walenz from 2003-JAN-02 to 2004-APR-07 * are Copyright 2003-2004 Applera Corporation, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2004-MAR-25 to 2004-OCT-10 * are Copyright 2004 Brian P. Walenz, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-MAY-23 to 2014-APR-11 * are Copyright 2005-2011,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2015-MAY-29 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef MERYL_H #define MERYL_H #include "AS_global.H" #include "AS_UTL_fileIO.H" #include "kMer.H" #include "bitPackedFile.H" #include "libmeryl.H" #include "speedCounter.H" #include "timeAndSize.H" #include #define PERSONALITY_MERGE 0xff #define PERSONALITY_MIN 0x01 #define PERSONALITY_MINEXIST 0x02 #define PERSONALITY_MAX 0x03 #define PERSONALITY_MAXEXIST 0x04 #define PERSONALITY_ADD 0x05 #define PERSONALITY_SUB 0x06 #define PERSONALITY_DIVIDE 0x07 #define PERSONALITY_ABS 0x08 #define PERSONALITY_AND 0x10 #define PERSONALITY_NAND 0x11 #define PERSONALITY_OR 0x12 #define PERSONALITY_XOR 0x13 #define PERSONALITY_LEQ 0x14 #define PERSONALITY_GEQ 0x15 #define PERSONALITY_EQ 0x16 class merylArgs { public: merylArgs(int argc, char **argv); merylArgs(const char *prefix); ~merylArgs(); void usage(void); void clear(void); uint64 hash(kMer const &mer) { return(mer.startOfMer(numBuckets_log2)); }; bool writeConfig(void); bool readConfig(const char *prefix); public: char *execName; char *options; bool beVerbose; bool doForward; bool doReverse; bool doCanonical; char *inputFile; char *outputFile; char *queryFile; uint32 merSize; uint32 merComp; bool positionsEnabled; uint64 numMersEstimated; uint64 numMersActual; uint64 numBasesActual; uint64 mersPerBatch; uint64 basesPerBatch; uint64 numBuckets; uint32 numBuckets_log2; uint32 merDataWidth; uint64 merDataMask; uint32 bucketPointerWidth; uint32 numThreads; uint64 memoryLimit; uint64 segmentLimit; bool configBatch; bool countBatch; bool mergeBatch; uint32 batchNumber; char *sgeJobName; char *sgeBuildOpt; char *sgeMergeOpt; bool isOnGrid; uint32 lowCount; uint32 highCount; uint32 desiredCount; bool outputCount; bool outputAll; bool outputPosition; bool includeDefLine; bool includeMer; uint32 mergeFilesMax; uint32 mergeFilesLen; char **mergeFiles; uint32 personality; }; uint64 estimateNumMersInMemorySize(uint32 merSize, uint64 mem, uint32 numThreads, bool positionsEnabled, bool beVerbose); uint64 estimateMemory(uint32 merSize, uint64 numMers, bool positionsEnabled); uint32 optimalNumberOfBuckets(uint32 merSize, uint64 numMers, bool positionsEnabled); void estimate(merylArgs *args); void build(merylArgs *args); void multipleOperations(merylArgs *args); void binaryOperations(merylArgs *args); void unaryOperations(merylArgs *args); void dump(merylArgs *args); void dumpThreshold(merylArgs *args); void dumpPositions(merylArgs *args); void countUnique(merylArgs *args); void dumpDistanceBetweenMers(merylArgs *args); void plotHistogram(merylArgs *args); #endif // MERYL_H canu-1.3/src/meryl/meryl.mk000066400000000000000000000013271272335422300157110ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := meryl SOURCES := meryl-args.C \ meryl-binaryOp.C \ meryl-build.C \ meryl-build-threads.C \ meryl-dump.C \ meryl-estimate.C \ meryl-merge.C \ meryl-unaryOp.C \ meryl.C SRC_INCDIRS := .. ../AS_UTL libleaff TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lleaff -lcanu TGT_PREREQS := libleaff.a libcanu.a SUBMAKEFILES := canu-1.3/src/meryl/simple.C000066400000000000000000000143721272335422300156310ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * kmer/meryl/simple.C * * Modifications by: * * Brian P. Walenz from 2010-AUG-31 to 2014-APR-11 * are Copyright 2010-2011,2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-05 to 2015-APR-24 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "AS_UTL_fileIO.H" #include "kMer.H" #include "bitPackedFile.H" #include "libmeryl.H" #include "seqStream.H" #include "merStream.H" #include "speedCounter.H" #include "speedCounter.H" #include "timeAndSize.H" #include using namespace std; #if 0 #include #include #include #include #include #include "bio++.H" #include "meryl.H" #include "libmeryl.H" #include "seqStream.H" #include "merStream.H" #endif // A very simple mer counter. Allocates a gigantic 32-bit array, // populates the array with mers, sorts, writes output. int main(int argc, char **argv) { char *inName = 0L; char *otName = 0L; uint32 merSize = 22; uint32 merCompression = 1; bool doForward = false; bool doReverse = false; bool doCanonical = false; speedCounter *C = 0L; merStream *M = 0L; merylStreamWriter *W = 0L; uint64 numMers = 0; bool algSorting = false; int arg = 1; int err = 0; while (arg < argc) { if (strcmp(argv[arg], "-i") == 0) { inName = argv[++arg]; } else if (strcmp(argv[arg], "-o") == 0) { otName = argv[++arg]; } else if (strcmp(argv[arg], "-m") == 0) { merSize = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-f") == 0) { doForward = true; } else if (strcmp(argv[arg], "-r") == 0) { doReverse = true; } else if (strcmp(argv[arg], "-C") == 0) { doCanonical = true; } else if (strcmp(argv[arg], "-c") == 0) { merCompression = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-sort") == 0) { algSorting = true; } else if (strcmp(argv[arg], "-direct") == 0) { algSorting = false; } else { fprintf(stderr, "unknown option '%s'\n", argv[arg]); err++; } arg++; } if (inName == 0L) { fprintf(stderr, "no input given with '-i'\n"); err++; } if (otName == 0L) { fprintf(stderr, "no output given with '-o'\n"); err++; } if (err) exit(1); M = new merStream(new kMerBuilder(merSize, merCompression), new seqStream(inName), true, true); numMers = M->approximateNumberOfMers(); delete M; fprintf(stderr, "Guessing "F_U64" mers in input '%s'\n", numMers, inName); M = new merStream(new kMerBuilder(merSize, merCompression), new seqStream(inName), true, true); if (algSorting) { uint64 theMersLen = 0; uint64 theMersMax = 2 * numMers; // for allowing both -f and -r uint32 *theMers = new uint32 [theMersMax]; fprintf(stderr, "Allocating "F_U64"MB for mer storage.\n", numMers * sizeof(uint64) >> 20); C = new speedCounter(" Filling mer list: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, 1); while (M->nextMer()) { if (doForward) theMers[theMersLen++] = M->theFMer(); if (doReverse) theMers[theMersLen++] = M->theRMer(); if (doCanonical) theMers[theMersLen++] = (M->theFMer() <= M->theRMer()) ? M->theFMer() : M->theRMer(); C->tick(); } delete C; delete M; fprintf(stderr, "Found "F_U64" mers in input '%s'\n", theMersLen, inName); if (theMersLen > theMersMax) fprintf(stderr, "ERROR: too many mers in input!\n"), exit(1); sort(theMers, theMers + theMersLen); C = new speedCounter(" Writing output: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, 1); W = new merylStreamWriter(otName, merSize, merCompression, 16, false); kMer mer(merSize); for (uint64 i=0; iaddMer(mer, 1, 0L); C->tick(); } delete C; delete W; delete [] theMers; } else { uint64 numCounts = ((uint64)1) << (2 * merSize); uint32 *theCounts = new uint32 [numCounts]; fprintf(stderr, "Allocating "F_U64"MB for count storage.\n", numCounts * sizeof(uint32) >> 20); memset(theCounts, 0, sizeof(uint32) * numCounts); C = new speedCounter(" Filling mer counts: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, 1); while (M->nextMer()) { if (doForward) theCounts[M->theFMer()]++; if (doReverse) theCounts[M->theRMer()]++; if (doCanonical) theCounts[(M->theFMer() <= M->theRMer()) ? M->theFMer() : M->theRMer()]++; C->tick(); } delete C; delete M; C = new speedCounter(" Writing output: %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, 1); W = new merylStreamWriter(otName, merSize, merCompression, 16, false); kMer mer(merSize); for (uint64 i=0; i 0) { mer.setWord(0, i); W->addMer(mer, theCounts[i], 0L); C->tick(); } } delete C; delete W; delete [] theCounts; } exit(0); } canu-1.3/src/meryl/simple.mk000066400000000000000000000007531272335422300160540ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := simple SOURCES := simple.C SRC_INCDIRS := .. ../AS_UTL libleaff TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lleaff -lcanu TGT_PREREQS := libleaff.a libcanu.a SUBMAKEFILES := canu-1.3/src/meryl/test/000077500000000000000000000000001272335422300152045ustar00rootroot00000000000000canu-1.3/src/meryl/test/Makefile000066400000000000000000000025401272335422300166450ustar00rootroot00000000000000PROG = stupidcount exhaustive INCLUDE = -I.. -I../../libutil -I../../libbio -I../../libmeryl LIBS = -L.. -L../../libutil -L../../libbio -L../../libmeryl -lmeryl -lbio -lutil -lm MERSIZE = 26 include ../../Make.compilers all: $(PROG) test-reduce stupidcount: stupidcount.C $(CXX) $(CXXFLAGS_COMPILE) -c -o stupidcount.o stupidcount.C $(INCLUDE) $(CXX) $(CXXLDFLAGS) -o stupidcount stupidcount.o $(LIBS) exhaustive: exhaustive.C kmerlite.H $(CXX) $(CXXFLAGS_COMPILE) -c -o exhaustive.o exhaustive.C $(INCLUDE) $(CXX) $(CXXLDFLAGS) -o exhaustive exhaustive.o $(LIBS) test-exhaustive: exhaustive ../meryl ../../leaff/leaff ../../leaff/leaff -G 1000 10000 40000 > g.fasta ../meryl -B -s g.fasta -o s -m $(MERSIZE) -threads 7 ./exhaustive -m s -f g.fasta test-reduce: ../meryl ../meryl -B -f -m 20 -s test-seq1.fasta -o 1 # Build the initial table ../meryl -Dt -n 0 -s 1 > 2.reduce.fasta # Dump the initial table as fasta ../meryl -B -f -m 20 -s 2.reduce.fasta -o 2 # Build a new table on the dumped fasta ../meryl -M sub -s 1 -s 2 -o 3 # Remove one copy of each mer ../meryl -Dt -n 1 -s 3 # Dump the resulting file echo 1 10 9 1 is correct touch test-reduce test: ../meryl -B -s test-seq1.fasta -o t -m 20 clean: rm -f $(PROG) *.o *.mc??? test-reduce *.seqStore* g.fasta 2.reduce.fasta *.fastaidx canu-1.3/src/meryl/test/exhaustive.C000066400000000000000000000140561272335422300175030ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "bio++.H" #include "libmeryl.H" #include "kmerlite.H" // This tests that all the mers in an input fasta file are counted // properly. It does not test that the meryl output contains exactly // those mers, just that those mers are there. // // If you can fit into one batch, then it _will_ verift that the // meryl output is exactly correct. // // Reads a meryl-format kmer count in chunks. Each chunk is stored // in a searchable structure (we should be using, say, an extended // existDB, but we're using a balanced binary tree). The entire // source fasta file is then streamed against the kmer chunk, // decrementing the count for each mer. When the whole file is // streamed, any kmers with positive count are reported. // NB: My hacked kazlib returns a pointer to whatever we give it. // Since we gave it a pointer to an object, it gives us back a // pointer to "a pointer to an object". Hence, this ugliness. // int kMerLiteSort(void const *a, void const *b) { kMerLite const *A = *((kMerLite * const *)a); kMerLite const *B = *((kMerLite * const *)b); if (*A < *B) return(-1); if (*A > *B) return(1); return(0); } int main(int argc, char **argv) { char *merylCount = 0L; char *fastaName = 0L; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-m") == 0) { merylCount = argv[++arg]; } else if (strcmp(argv[arg], "-f") == 0) { fastaName = argv[++arg]; } else { fprintf(stderr, "unknown option '%s'\n", argv[arg]); } arg++; } if ((merylCount == 0L) || (fastaName == 0L)) { fprintf(stderr, "usage: %s -m -f \n", argv[0]); exit(1); } // Open the count files // merylStreamReader *MSR = new merylStreamReader(merylCount); fprintf(stderr, "Mers are "uint32FMT" bases.\n", MSR->merSize()); fprintf(stderr, "There are "uint64FMT" unique (copy = 1) mers.\n", MSR->numberOfUniqueMers()); fprintf(stderr, "There are "uint64FMT" distinct mers.\n", MSR->numberOfDistinctMers()); fprintf(stderr, "There are "uint64FMT" mers total.\n", MSR->numberOfTotalMers()); // Guess how many mers we can fit into 512MB, then report how many chunks we need to do. uint32 merSize = MSR->merSize(); uint64 memoryLimit = 700 * 1024 * 1024; uint64 perMer = sizeof(kMerLite) + sizeof(dnode_t); uint64 mersPerBatch = memoryLimit / perMer; uint32 numBatches = MSR->numberOfDistinctMers() / mersPerBatch; uint32 batch = 0; dnode_t *nodes = new dnode_t [mersPerBatch]; kMerLite *mers = new kMerLite [mersPerBatch]; if (MSR->numberOfDistinctMers() % mersPerBatch) numBatches++; fprintf(stderr, "perMer: "uint64FMT" bytes ("uint64FMT" for kMerLite, "uint64FMT" for dnode_t.\n", perMer, (uint64)sizeof(kMerLite), (uint64)sizeof(dnode_t)); fprintf(stderr, "We can fit "uint64FMT" mers into "uint64FMT"MB.\n", mersPerBatch, memoryLimit >> 20); fprintf(stderr, "So we need "uint32FMT" batches to verify the count.\n", numBatches); while (MSR->validMer()) { uint64 mersRemain = mersPerBatch; dict_t *merDict = dict_create(mersPerBatch, kMerLiteSort); batch++; // STEP 1: Insert mersPerBatch into the merDict // fprintf(stderr, "STEP 1 BATCH "uint32FMTW(2)": Insert into merDict\n", batch); while (MSR->nextMer() && mersRemain) { mersRemain--; mers[mersRemain] = MSR->theFMer(); // initialize the node with the value, then insert the node // into the tree using the key int32 val = (int32)MSR->theCount(); dnode_init(&nodes[mersRemain], (void *)val); dict_insert(merDict, &nodes[mersRemain], &mers[mersRemain]); } // STEP 2: Stream the original file, decrementing the count // fprintf(stderr, "STEP 2 BATCH "uint32FMTW(2)": Stream fasta\n", batch); seqStream *CS = new seqStream(fastaName, true); merStream *MS = new merStream(new kMerBuilder(merSize), CS); kMerLite mer; dnode_t *nod; while (MS->nextMer()) { mer = MS->theFMer(); nod = dict_lookup(merDict, &mer); if (nod != 0L) { int32 val = (int32)dnode_get(nod); val--; dnode_put(nod, (void *)val); } else { // Unless the whole meryl file fit into our merDict, we cannot warn if // we don't find mers. // if (numBatches == 1) { char str[1024]; fprintf(stderr, "Didn't find node for mer '%s'\n", mer.merToString(merSize, str)); } } } delete MS; delete CS; // STEP 3: Check every node in the tree to make sure that the counts // are exactly zero. // fprintf(stderr, "STEP 3 BATCH "uint32FMTW(2)": Check\n", batch); nod = dict_first(merDict); while (nod) { int32 val = (int32)dnode_get(nod); kMerLite const *nodmer = (kMerLite const *)dnode_getkey(nod); if (val != 0) { char str[1024]; fprintf(stderr, "Got count "int32FMT" for mer '%s'\n", val, nodmer->merToString(merSize, str)); } nod = dict_next(merDict, nod); } // STEP 4: Destroy the dictionary. // fprintf(stderr, "STEP 4 BATCH "uint32FMTW(2)": Destroy\n", batch); while ((nod = dict_first(merDict))) dict_delete(merDict, nod); dict_destroy(merDict); } } canu-1.3/src/meryl/test/kmerlite.H000066400000000000000000000076451272335422300171450ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "bio++.H" #ifndef KMER_LITE_H #define KMER_LITE_H //////////////////////////////////////// // // This is kMerLite -- derived from kMer.H, removing // most of the accessors. // // Assumes that KMER_WORDS is already defined. class kMerLite { public: // Used by some of the test routines. void dump(void) const { for (uint32 i=0; i> 5; char *str = instr; if ((merSize & uint32MASK(6)) == 0) lastWord++; // We build the string right to left, print any partial word // first, then print whole words until we run out of words to // print. if (merSize & uint32MASK(5)) { ::merToString(merSize & uint32MASK(5), _wd[lastWord], str); str += merSize & uint32MASK(5); } while (lastWord > 0) { lastWord--; ::merToString(32, _wd[lastWord], str); str += 32; } return(instr); }; #if KMER_WORDS == 1 bool operator!=(kMerLite const &r) const { return(_wd[0] != r._wd[0]); }; bool operator==(kMerLite const &r) const { return(_wd[0] == r._wd[0]); }; bool operator<(kMerLite const &r) const { return(_wd[0] < r._wd[0]); }; bool operator>(kMerLite const &r) const { return(_wd[0] > r._wd[0]); }; bool operator<=(kMerLite const &r) const { return(_wd[0] <= r._wd[0]); }; bool operator>=(kMerLite const &r) const { return(_wd[0] >= r._wd[0]); }; #else bool operator!=(kMerLite const &r) const { uint64 res = uint64ZERO; for (uint32 i=KMER_WORDS; i--; ) res |= _wd[i] ^ r._wd[i]; return(res != uint64ZERO); }; bool operator==(kMerLite const &r) const { uint64 res = uint64ZERO; for (uint32 i=KMER_WORDS; i--; ) res |= _wd[i] ^ r._wd[i]; return(res == uint64ZERO); }; bool operator<(kMerLite const &r) const { for (uint32 i=KMER_WORDS; i--; ) { if (_wd[i] < r._wd[i]) return(true); if (_wd[i] > r._wd[i]) return(false); } return(false); }; bool operator>(kMerLite const &r) const { for (uint32 i=KMER_WORDS; i--; ) { if (_wd[i] > r._wd[i]) return(true); if (_wd[i] < r._wd[i]) return(false); } return(false); }; bool operator<=(kMerLite const &r) const { for (uint32 i=KMER_WORDS; i--; ) { if (_wd[i] < r._wd[i]) return(true); if (_wd[i] > r._wd[i]) return(false); } return(true); }; bool operator>=(kMerLite const &r) const { for (uint32 i=KMER_WORDS; i--; ) { if (_wd[i] > r._wd[i]) return(true); if (_wd[i] < r._wd[i]) return(false); } return(true); }; #endif private: uint64 _wd[KMER_WORDS]; }; #endif // KMER_LITE_H canu-1.3/src/meryl/test/stupidcount.C000066400000000000000000000032101272335422300176650ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "bio++.H" // Reads a sequence file, outputs a list of the mers in it. You can // then pipe this to unix sort and uniq to do a mercount. You // probably don't want to count large things this way... int main(int argc, char **argv) { char *seqName = 0L; uint32 merSize = 20; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-s") == 0) { seqName = argv[++arg]; } else if (strcmp(argv[arg], "-m") == 0) { merSize = strtouint32(argv[++arg], 0L); } arg++; } if (seqName == 0L) { fprintf(stderr, "usage: %s [-m mersize] -s seqfile.fasta\n", argv[0]); exit(1); } seqStream *CS = new seqStream(seqName, true); merStream *MS = new merStream(new kMerBuilder(merSize), CS); char str[1024]; while (MS->nextMer()) fprintf(stdout, "%s\n", MS->theFMer().merToString(str)); delete MS; delete CS; exit(0); } canu-1.3/src/meryl/test/test-seq1.fasta000066400000000000000000000005321272335422300200520ustar00rootroot00000000000000> 1 A 20 CG 0 T AAAAAAAAAAAAAAAAAAAAGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCG > 1 A 1 CG 0 T ----not-a-mer------ -----is-a-mer------- AAAAAAAAAAAAAAAAAAAANGCGCGCGCGCGCGCGCGCGNCGCGCGCGCGCGCGCGCGCG > (zero 63 bases) ATNCGGATYCGATCGASCHJAGSVHYWERIGHWEEIRVHSDKFVHWIERVHIWRVHKSDFVKS > 20 T NNNNTTTTTTTTTTTTTTTTTTTTNNNNTTTTTTTTTTTTTTTTTTTTNNNN canu-1.3/src/meryl/test/test-seq2.fasta000066400000000000000000000010711272335422300200520ustar00rootroot00000000000000> 2 A 20 CG 0 T AAAAAAAAAAAAAAAAAAAAN AAAAAAAAAAAAAAAAAAAAN AAAAAAAAAAAAAAAAAAAAN AAAAAAAAAAAAAAAAAAAAN AAAAAAAAAAAAAAAAAAAAN AAAAAAAAAAAAAAAAAAACN AAAAAAAAAAAAAAAAAAACN AAAAAAAAAAAAAAAAAAACN AAAAAAAAAAAAAAAAAAACN AAAAAAAAAAAAAAAAAAACN AAAAAAAAAAAAAAAAAAAAAAAGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCGCG > 1 A 1 CG 0 T ----not-a-mer------ -----is-a-mer------- AAAAAAAAAAAAAAAAAAAANGCGCGCGCGCGCGCGCGCGNCGCGCGCGCGCGCGCGCGCG > (zero 63 bases) ATNCGGATYCGATCGASCHJAGSVHYWERIGHWEEIRVHSDKFVHWIERVHIWRVHKSDFVKS > 20 T NNNNTTTTTTTTTTTTTTTTTTTTNNNNTTTTTTTTTTTTTTTTTTTTNNNN canu-1.3/src/meryl/test/test-seq3.fasta000066400000000000000000000000741272335422300200550ustar00rootroot00000000000000> ACGCTCAGCTACTACGACTTAGAGAAAATAGCGATATAGCGATCGATCGATTAGAGA canu-1.3/src/mhap/000077500000000000000000000000001272335422300140225ustar00rootroot00000000000000canu-1.3/src/mhap/mhap.mk000066400000000000000000000005341272335422300153020ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := mhap-2.1.jar SOURCES := mhap-2.1.tar canu-1.3/src/mhap/mhapConvert.C000066400000000000000000000077611272335422300164270ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAR-27 to 2015-JUN-25 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Sergey Koren beginning on 2016-FEB-24 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "ovStore.H" #include "splitToWords.H" #include using namespace std; int main(int argc, char **argv) { bool asCoords = true; char *outName = NULL; uint32 baseIDhash = 0; uint32 numIDhash = 0; uint32 baseIDquery = 0; vector files; int32 arg = 1; int32 err = 0; while (arg < argc) { if (strcmp(argv[arg], "-o") == 0) { outName = argv[++arg]; } else if (strcmp(argv[arg], "-h") == 0) { baseIDhash = atoi(argv[++arg]) - 1; numIDhash = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-q") == 0) { baseIDquery = atoi(argv[++arg]) - 1; } else if (AS_UTL_fileExists(argv[arg])) { files.push_back(argv[arg]); } else { fprintf(stderr, "ERROR: invalid arg '%s'\n", argv[arg]); err++; } arg++; } if ((err) || (files.size() == 0)) { fprintf(stderr, "usage: %s [options] file.mhap[.gz]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " Converts mhap native output to ovb\n"); fprintf(stderr, "\n"); fprintf(stderr, " -o out.ovb output file\n"); fprintf(stderr, "\n"); fprintf(stderr, " -h id num base id and number of hash table reads\n"); fprintf(stderr, " (mhap output IDs 1 through 'num')\n"); fprintf(stderr, " -q id base id of query reads\n"); fprintf(stderr, " (mhap output IDs 'num+1' and higher)\n"); if (files.size() == 0) fprintf(stderr, "ERROR: no overlap files supplied\n"); exit(1); } char *ovStr = new char [1024]; ovOverlap ov(NULL); ovFile *of = new ovFile(outName, ovFileFullWrite); for (uint32 ff=0; fffile()) != NULL) { splitToWords W(ovStr); ov.a_iid = W(0) + baseIDquery - numIDhash; // First ID is the query ov.b_iid = W(1) + baseIDhash; // Second ID is the hash table if (ov.a_iid == ov.b_iid) continue; assert(W[4][0] == '0'); ov.dat.ovl.forUTG = true; ov.dat.ovl.forOBT = true; ov.dat.ovl.forDUP = true; ov.dat.ovl.ahg5 = W(5); ov.dat.ovl.ahg3 = W(7) - W(6); if (W[8][0] == '0') { ov.dat.ovl.bhg5 = W(9); ov.dat.ovl.bhg3 = W(11) - W(10); ov.flipped(false); } else { ov.dat.ovl.bhg3 = W(9); ov.dat.ovl.bhg5 = W(11) - W(10); ov.flipped(true); } ov.erate(atof(W[2])); of->writeOverlap(&ov); } arg++; } delete of; delete [] ovStr; exit(0); } canu-1.3/src/mhap/mhapConvert.mk000066400000000000000000000007541272335422300166470ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := mhapConvert SOURCES := mhapConvert.C SRC_INCDIRS := .. ../AS_UTL ../stores liboverlap TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/minimap/000077500000000000000000000000001272335422300145275ustar00rootroot00000000000000canu-1.3/src/minimap/mmapConvert.C000066400000000000000000000066511272335422300171360ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2016-FEB-24 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "ovStore.H" #include "splitToWords.H" #include using namespace std; int main(int argc, char **argv) { bool asCoords = true; char *outName = NULL; vector files; int32 arg = 1; int32 err = 0; while (arg < argc) { if (strcmp(argv[arg], "-o") == 0) { outName = argv[++arg]; } else if (AS_UTL_fileExists(argv[arg])) { files.push_back(argv[arg]); } else { fprintf(stderr, "ERROR: invalid arg '%s'\n", argv[arg]); err++; } arg++; } if ((err) || (files.size() == 0)) { fprintf(stderr, "usage: %s [options] file.mhap[.gz]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " Converts mhap native output to ovb\n"); fprintf(stderr, "\n"); fprintf(stderr, " -o out.ovb output file\n"); fprintf(stderr, "\n"); if (files.size() == 0) fprintf(stderr, "ERROR: no overlap files supplied\n"); exit(1); } char *ovStr = new char [1024]; ovOverlap ov(NULL); ovFile *of = new ovFile(outName, ovFileFullWrite); for (uint32 ff=0; fffile()) != NULL) { splitToWords W(ovStr); ov.a_iid = W(0); ov.b_iid = W(5); if (ov.a_iid == ov.b_iid) continue; ov.dat.ovl.forUTG = true; ov.dat.ovl.forOBT = true; ov.dat.ovl.forDUP = true; ov.dat.ovl.ahg5 = W(2); ov.dat.ovl.ahg3 = W(1) - W(3); if (W[4][0] == '+') { ov.dat.ovl.bhg5 = W(7); ov.dat.ovl.bhg3 = W(6) - W(8); ov.flipped(false); } else { ov.dat.ovl.bhg3 = W(7); ov.dat.ovl.bhg5 = W(6) - W(8); ov.flipped(true); } ov.erate(1-((double)W(9)/W(10))); of->writeOverlap(&ov); } arg++; } delete of; delete [] ovStr; exit(0); } canu-1.3/src/minimap/mmapConvert.mk000066400000000000000000000007541272335422300173610ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := mmapConvert SOURCES := mmapConvert.C SRC_INCDIRS := .. ../AS_UTL ../stores liboverlap TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/overlapBasedTrimming/000077500000000000000000000000001272335422300172135ustar00rootroot00000000000000canu-1.3/src/overlapBasedTrimming/adjustFlipped.C000066400000000000000000000111461272335422300221200ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-APR-15 to 2015-JUN-16 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "adjustOverlaps.H" // Adjust the overlap for any trimming done already. This works by computing the fraction of the // overlap trimmed for each read and each end, picking the largest fraction for each end, and // applying that fraction to the other read. // // It expects only flipped overlaps. The output coordinates are for a REVERSE COMPLEMENTED // b read. if you care which end is the actual 5' or 3' end, look at flipped(). bool adjustFlipped(clearRangeFile *iniClr, gkStore *gkp, ovOverlap *ovl, uint32 &aovlbgn, uint32 &aovlend, uint32 &bovlbgn, uint32 &bovlend, uint32 &aclrbgn, uint32 &aclrend, uint32 &bclrbgn, uint32 &bclrend) { assert(ovl->flipped() == true); uint32 bLen = gkp->gkStore_getRead(ovl->b_iid)->gkRead_sequenceLength(); aovlbgn = ovl->a_bgn(); bovlbgn = bLen - ovl->b_bgn(); // bgn(), because this is the higher coord aovlend = ovl->a_end(); bovlend = bLen - ovl->b_end(); aclrbgn = iniClr->bgn(ovl->a_iid); bclrbgn = bLen - iniClr->end(ovl->b_iid); // end(), because this is the higher coord aclrend = iniClr->end(ovl->a_iid); bclrend = bLen - iniClr->bgn(ovl->b_iid); assert(aovlbgn < aovlend); assert(bovlbgn < bovlend); if ((aclrend <= aovlbgn) || (aovlend <= aclrbgn) || (bclrend <= bovlbgn) || (bovlend <= bclrbgn)) { // Overlap doesn't intersect clear range, fail. #if 0 fprintf(stderr, "Discard FLIP overlap from %u,%u-%u,%u based on clear ranges %u,%u and %u,%u\n", aovlbgn, aovlend, bovlbgn, bovlend, aclrbgn, aclrend, bclrbgn, bclrend); #endif return(false); } uint32 alen = aovlend - aovlbgn; uint32 blen = bovlend - bovlbgn; double afracbgn = (double)((aclrbgn < aovlbgn) ? (0) : (aclrbgn - aovlbgn)) / alen; double bfracbgn = (double)((bclrbgn < bovlbgn) ? (0) : (bclrbgn - bovlbgn)) / blen; double afracend = (double)((aclrend > aovlend) ? (0) : (aovlend - aclrend)) / alen; double bfracend = (double)((bclrend > bovlend) ? (0) : (bovlend - bclrend)) / blen; //fprintf(stderr, "frac a %.20f %.20f b %.20f %.20f\n", afracbgn, afracend, bfracbgn, bfracend); //fprintf(stderr, "frac a %.20f %.20f b %.20f %.20f\n", afracbgn * alen, afracend * alen, bfracbgn * blen, bfracend * blen); double maxbgn = max(afracbgn, bfracbgn); double maxend = max(afracend, bfracend); //fprintf(stderr, "frac a %.20f %.20f b %.20f %.20f\n", maxbgn * alen, maxend * alen, maxbgn * blen, maxend * blen); assert(maxbgn < 1.0); assert(maxend < 1.0); uint32 aadjbgn = (uint32)round(maxbgn * alen); uint32 badjbgn = (uint32)round(maxbgn * blen); uint32 aadjend = (uint32)round(maxend * alen); uint32 badjend = (uint32)round(maxend * blen); //fprintf(stderr, "frac a %u %u b %u %u alen %u blen %u\n", aadjbgn, aadjend, badjbgn, badjend, alen, blen); #if 0 fprintf(stderr, "Adjusted FLIP overlap from %u,%u-%u,%u (adjust %u,%u,%u,%u) to %u,%u-%u,%u based on clear ranges %u,%u and %u,%u maxbgn=%f maxend=%f\n", aovlbgn, aovlend, bovlbgn, bovlend, aadjbgn, aadjend, badjbgn, badjend, aovlbgn + aadjbgn, aovlend - aadjend, bovlbgn + badjbgn, bovlend - badjend, aclrbgn, aclrend, bclrbgn, bclrend, maxbgn, maxend); #endif aovlbgn += aadjbgn; bovlbgn += badjbgn; aovlend -= aadjend; bovlend -= badjend; assert(aclrbgn <= aovlbgn); assert(bclrbgn <= bovlbgn); assert(aovlend <= aclrend); assert(bovlend <= bclrend); return(true); } canu-1.3/src/overlapBasedTrimming/adjustNormal.C000066400000000000000000000103711272335422300217640ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-APR-15 to 2015-JUN-16 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "adjustOverlaps.H" // Adjust the overlap for any trimming done already. This works by computing the fraction of the // overlap trimmed for each read and each end, picking the largest fraction for each end, and // applying that fraction to the other read. // // It expects only normal overlaps. bool adjustNormal(clearRangeFile *iniClr, gkStore *gkp, ovOverlap *ovl, uint32 &aovlbgn, uint32 &aovlend, uint32 &bovlbgn, uint32 &bovlend, uint32 &aclrbgn, uint32 &aclrend, uint32 &bclrbgn, uint32 &bclrend) { assert(ovl->flipped() == false); aovlbgn = ovl->a_bgn(); bovlbgn = ovl->b_bgn(); aovlend = ovl->a_end(); bovlend = ovl->b_end(); aclrbgn = iniClr->bgn(ovl->a_iid); bclrbgn = iniClr->bgn(ovl->b_iid); aclrend = iniClr->end(ovl->a_iid); bclrend = iniClr->end(ovl->b_iid); assert(aovlbgn < aovlend); assert(bovlbgn < bovlend); if ((aclrend <= aovlbgn) || (aovlend <= aclrbgn) || (bclrend <= bovlbgn) || (bovlend <= bclrbgn)) { // Overlap doesn't intersect clear range, fail. #if 0 fprintf(stderr, "Discard NORM overlap from %u,%u-%u,%u based on clear ranges %u,%u and %u,%u\n", aovlbgn, aovlend, bovlbgn, bovlend, aclrbgn, aclrend, bclrbgn, bclrend); #endif return(false); } uint32 alen = aovlend - aovlbgn; uint32 blen = bovlend - bovlbgn; double afracbgn = (double)((aclrbgn < aovlbgn) ? (0) : (aclrbgn - aovlbgn)) / alen; double bfracbgn = (double)((bclrbgn < bovlbgn) ? (0) : (bclrbgn - bovlbgn)) / blen; double afracend = (double)((aclrend > aovlend) ? (0) : (aovlend - aclrend)) / alen; double bfracend = (double)((bclrend > bovlend) ? (0) : (bovlend - bclrend)) / blen; //fprintf(stderr, "frac a %.20f %.20f b %.20f %.20f\n", afracbgn, afracend, bfracbgn, bfracend); //fprintf(stderr, "frac a %.20f %.20f b %.20f %.20f\n", afracbgn * alen, afracend * alen, bfracbgn * blen, bfracend * blen); double maxbgn = max(afracbgn, bfracbgn); double maxend = max(afracend, bfracend); //fprintf(stderr, "frac a %.20f %.20f b %.20f %.20f\n", maxbgn * alen, maxend * alen, maxbgn * blen, maxend * blen); assert(maxbgn < 1.0); assert(maxend < 1.0); uint32 aadjbgn = (uint32)round(maxbgn * alen); uint32 badjbgn = (uint32)round(maxbgn * blen); uint32 aadjend = (uint32)round(maxend * alen); uint32 badjend = (uint32)round(maxend * blen); //fprintf(stderr, "frac a %u %u b %u %u alen %u blen %u\n", aadjbgn, aadjend, badjbgn, badjend, alen, blen); #if 0 fprintf(stderr, "Adjusted NORM overlap from %u,%u-%u,%u (adjust %u,%u,%u,%u) to %u,%u-%u,%u based on clear ranges %u,%u and %u,%u maxbgn=%f maxend=%f\n", aovlbgn, aovlend, bovlbgn, bovlend, aadjbgn, aadjend, badjbgn, badjend, aovlbgn + aadjbgn, aovlend - aadjend, bovlbgn + badjbgn, bovlend - badjend, aclrbgn, aclrend, bclrbgn, bclrend, maxbgn, maxend); #endif aovlbgn += aadjbgn; bovlbgn += badjbgn; aovlend -= aadjend; bovlend -= badjend; assert(aclrbgn <= aovlbgn); assert(bclrbgn <= bovlbgn); assert(aovlend <= aclrend); assert(bovlend <= bclrend); return(true); } canu-1.3/src/overlapBasedTrimming/adjustOverlaps.H000066400000000000000000000031471272335422300223370ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2015-JUN-16 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef ADJUST_OVERLAPS_H #define ADJUST_OVERLAPS_H #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include "clearRangeFile.H" bool adjustNormal(clearRangeFile *iniClr, gkStore *gkp, ovOverlap *ovl, uint32 &aovlbgn, uint32 &aovlend, uint32 &bovlbgn, uint32 &bovlend, uint32 &aclrbgn, uint32 &aclrend, uint32 &bclrbgn, uint32 &bclrend); bool adjustFlipped(clearRangeFile *iniClr, gkStore *gkp, ovOverlap *ovl, uint32 &aovlbgn, uint32 &aovlend, uint32 &bovlbgn, uint32 &bovlend, uint32 &aclrbgn, uint32 &aclrend, uint32 &bclrbgn, uint32 &bclrend); #endif canu-1.3/src/overlapBasedTrimming/clearRangeFile.H000066400000000000000000000104521272335422300221710ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2015-APR-15 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef CLEAR_RANGE_FILE_H #define CLEAR_RANGE_FILE_H #include "AS_global.H" #include "gkStore.H" #include "AS_UTL_fileIO.H" class clearRangeFile { public: // Create a clear range file. If the file doesn't exist, maxID must be set to the number // of reads allowed. If the file does exist, we don't care and load the number of reads // from the file. clearRangeFile(char *fileName, gkStore *gkp) { strcpy(_fileName, fileName); if (AS_UTL_fileExists(_fileName) == false) { _modified = true; // Always gets written, regardless of changes _lastID = gkp->gkStore_getNumReads(); _bgn = new uint32 [_lastID + 1]; _end = new uint32 [_lastID + 1]; // Every range is undefined memset(_bgn, 0, sizeof(uint32) * (_lastID + 1)); memset(_end, 0, sizeof(uint32) * (_lastID + 1)); // Nope, every range is set to fully clear. reset(gkp); } else { _modified = false; FILE *F = fopen(_fileName, "r"); if (errno) fprintf(stderr, "clearRangeFile()-- Failed to open '%s' for loading clear ranges: %s\n", _fileName, strerror(errno)), exit(1); AS_UTL_safeRead(F, &_lastID, "clearRangeFile::lastID", sizeof(uint32), 1); assert(gkp->gkStore_getNumReads() == _lastID); // Sane? _bgn = new uint32 [_lastID + 1]; _end = new uint32 [_lastID + 1]; AS_UTL_safeRead(F, _bgn, "clearRangeFile::bgn", sizeof(uint32), _lastID + 1); AS_UTL_safeRead(F, _end, "clearRangeFile::end", sizeof(uint32), _lastID + 1); fclose(F); } }; // Close the clear range file. ~clearRangeFile() { if (_modified == true) { FILE *F = fopen(_fileName, "w"); if (errno) fprintf(stderr, "clearRangeFile()-- Failed to open '%s' for saving clear ranges: %s\n", _fileName, strerror(errno)), exit(1); AS_UTL_safeWrite(F, &_lastID, "clearRangeFile::lastID", sizeof(uint32), 1); AS_UTL_safeWrite(F, _bgn, "clearRangeFile::bgn", sizeof(uint32), _lastID + 1); AS_UTL_safeWrite(F, _end, "clearRangeFile::end", sizeof(uint32), _lastID + 1); fclose(F); } delete [] _bgn; delete [] _end; }; void reset(gkStore *gkp) { for (uint32 fi=1; fi <= _lastID; fi++) { _bgn[fi] = 0; _end[fi] = gkp->gkStore_getRead(fi)->gkRead_sequenceLength(); } }; uint32 bgn(uint32 id) { assert(id <= _lastID); return(_bgn[id]); }; uint32 end(uint32 id) { assert(id <= _lastID); return(_end[id]); }; uint32 &setbgn(uint32 id) { assert(id <= _lastID); _modified = true; return(_bgn[id]); }; uint32 &setend(uint32 id) { assert(id <= _lastID); _modified = true; return(_end[id]); }; bool isDeleted(uint32 id) { assert(id <= _lastID); return((_bgn[id] == UINT32_MAX) && (_end[id] == UINT32_MAX)); }; void setDeleted(uint32 id) { assert(id <= _lastID); _modified = true; _bgn[id] = UINT32_MAX; _end[id] = UINT32_MAX; }; void copy(clearRangeFile *source) { if (source == NULL) return; assert(_lastID == source->_lastID); memcpy(_bgn, source->_bgn, sizeof(uint32) * (_lastID + 1)); memcpy(_end, source->_end, sizeof(uint32) * (_lastID + 1)); }; private: bool _modified; char _fileName[FILENAME_MAX]; uint32 _lastID; // [_lastID] is valid; allocated _lastID+1 spots. uint32 *_bgn; uint32 *_end; }; #endif // CLEAR_RANGE_FILE_H canu-1.3/src/overlapBasedTrimming/detect-unsplit-subreads-in-overlaps.pl000066400000000000000000000056041272335422300265640ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my %readLen; open(F, "gatekeeper -dumpfragments -tabular test.gkpStore |"); while () { my @v = split '\s+', $_; $readLen{$v[1]} = $v[9]; } close(F); my %readHash; my %readConf; my $lastRead = 0; my $nOvl = 0; my $potSub = 0; my $proSub = 0; open(F, "overlapStore -d test.ovlStore |"); while () { s/^\s+//; s/\s+$//; my @v = split '\s+', $_; if ($lastRead != $v[0]) { my $mCount = scalar(keys %readConf); if ($mCount > 0) { print "$lastRead\t$mCount\n"; $potSub++; } if ($mCount > 5) { $proSub++; } $lastRead = $v[0]; $nOvl = 0; undef %readHash; undef %readConf; } next if (($v[3] < 0) && ($v[4] > 0)); # read is contained in other $nOvl++; if (! exists($readHash{$v[1]})) { $readHash{$v[1]} = "$v[3] $v[4]"; next; } my ($na, $nb) = ($v[3], $v[4]); my ($oa, $ob) = split '\s+', $readHash{$v[1]}; my $len = $readLen{$v[0]}; my $nbgn = ($na > 0) ? $na : 0; my $nend = ($nb < 0) ? ($len + $nb) : $len; my $obgn = ($oa > 0) ? $oa : 0; my $oend = ($ob < 0) ? ($len + $ob) : $len; #print "$_ -- $nbgn $nend - $obgn $oend\n"; goto saveConf if (($nbgn <= $obgn) && ($nend <= $obgn)); # n before o goto saveConf if (($nbgn >= $oend) && ($nend >= $oend)); # n after o my $min = ($nbgn < $obgn) ? $obgn : $nbgn; # Largest bgn coord my $max = ($nend < $oend) ? $nend : $oend; # Smallest end coord my $nlen = $nend - $nbgn; my $olen = $oend - $obgn; my $olap = $max - $min; #print "$_ -- $olap $nlen $olen\n"; next if (($olap / $nlen > 0.5) || ($olap / $olen > 0.5)); # reads overlap by lots # different regions, probably sub read detected saveConf: $readConf{$v[1]}++; #print "$_\n"; } close(F); print STDERR "potential unsplit subreads: $potSub\n"; print STDERR "probable unsplit subreads: $proSub\n"; canu-1.3/src/overlapBasedTrimming/generate-random-chimeric-fragments.pl000066400000000000000000000112631272335422300263700ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## se strict; # ########################################################################### # # This file is part of Celera Assembler, a software program that # assembles whole-genome shotgun reads into contigs and scaffolds. # Copyright (C) 2009, J. Craig Venter Instititue. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received (LICENSE.txt) a copy of the GNU General Public # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # ########################################################################### # # Generates random chimera and spur fragments from a supplied reference # genome. # # User unfriendly. # # A spur: [junk][sequence] # A chimer: [junk][sequence][junk][sequence][junk] my $genome = shift @ARGV; my $readLength = 400; my $leaff = "/work/wgs/kmer/leaff/leaff"; my $cvtto = "perl /work/wgs/src/AS_MSG/convert-fasta-to-v2.pl"; die "Reference sequence '$genome' not found.\n" if (! -e $genome); # generate some random reads from the genome, both forward and reverse. system("$leaff --errors $readLength 10000 1 0.01 $genome > sample3.fasta"); system("$leaff -f sample3.fasta -R -C -W > sample1.fasta"); system("$leaff --errors $readLength 10000 1 0.01 $genome > sample2.fasta"); # generate random small bits of sequence from those samples; leaff can't do this nicely. my @sequences; open(F, "< sample1.fasta"); while (!eof(F)) { my $h = ; chomp $h; my $s = ; chomp $s; my $r = rand(); push @sequences, "$r\0seq\0$s"; } close(F); open(F, "< sample2.fasta"); while (!eof(F)) { my $h = ; chomp $h; my $s = ; chomp $s; my $r = rand(); push @sequences, "$r\0rev\0$s"; } close(F); @sequences = sort { $a <=> $b } @sequences; open(F, "> chimer.fasta"); open(R, "$leaff -G 100000 $readLength $readLength |"); my $iid = 0; my $def = ""; my $seq = ""; my $seqLen = 0; while (scalar(@sequences) > 0) { my $np = int(rand() * 3 + 1); my $gl; $iid++; $def = "$iid"; $seq = ""; $seqLen = 0; $gl = int(rand() * 200 - 50); # between -50 and 150, junk on the front if ($gl > 0) { my $h = ; my $s = ; $def .= "-jnk-$gl"; $seq .= substr($s, 0, $gl); $seqLen += $gl; } for (my $pp=0; $pp < $np; $pp++) { my $S = pop @sequences; my ($r, $h, $s) = split '\0', $S; my $l = int(rand() * 200 + 50); # between 50 and 250. $def .= "-$h-$l"; $seq .= substr($s, 0, $l); $seqLen += $l; if ($pp + 1 == $np) { $gl = 200; } else { $gl = 100; } $gl = int(rand() * $gl - 50); # between -50 and 50, junk in the middle if ($gl > 0) { my $h = ; my $s = ; $def .= "-jnk-$gl"; $seq .= substr($s, 0, $gl); $seqLen += $gl; } } print F ">$def\n$seq\n"; } close(F); system("tr 'ACGT' 'NNNN' < chimer.fasta > chimer.qv"); system("perl /work/wgs/src/AS_MSG/convert-fasta-to-v2.pl -l CHIMER -s chimer.fasta -q chimer.qv > chimer.frg"); unlink "sample1.fasta"; unlink "sample2.fasta"; unlink "sample3.fasta"; unlink "sample1.fastaidx"; unlink "sample2.fastaidx"; unlink "sample3.fastaidx"; unlink "chimer.fasta"; unlink "chimer.qv"; canu-1.3/src/overlapBasedTrimming/splitReads-subReads.C000066400000000000000000000272701272335422300232070ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "splitReads.H" // Examine overlaps for a specific pattern indicating a read that flips back on itself. // This requires multiple overlaps from the same read, in opposite orientation, to work. // // If this read isn't suspected to contain sub reads, all we can do is mark some // overlaps as junk. // // If this read is suspected to contain sub reads, we want to mark the junction region. uint32 intervalOverlap(uint32 b1, uint32 e1, uint32 b2, uint32 e2) { uint32 minmax = MIN(e1, e2); uint32 maxmin = MAX(b1, b2); if (minmax > maxmin) return(minmax - maxmin); return(0); } bool doCheckSubRead(gkStore *gkp, uint32 id) { gkRead *read = gkp->gkStore_getRead(id); gkLibrary *libr = gkp->gkStore_getLibrary(read->gkRead_libraryID()); return(libr->gkLibrary_checkForSubReads() == true); } // Populate w->blist with intervals where a suspected subread junction occurs. void detectSubReads(gkStore *gkp, workUnit *w, FILE *subreadFile, bool subreadFileVerbose) { assert(w->adjLen > 0); assert(doCheckSubRead(gkp, w->id) == true); map secondIdx; map numOlaps; bool largePalindrome = false; intervalList BAD; intervalList BADall; // Count the number of overlaps for each b_iid, and remember the last index. There are supposed to // be at most two overlaps per ID pair, so if we remember the last, and iterate through, we can // get both. for (uint32 ii=0; iiadjLen; ii++) { secondIdx[w->adj[ii].b_iid] = ii; numOlaps [w->adj[ii].b_iid]++; } // Scan overlaps. For any pair of b_iid, with overlaps in opposite directions, compute a 'bad' // interval where a suspected flip occurs. for (uint32 ii=0; iiadjLen; ii++) { adjOverlap *aii = w->adj + ii; if (numOlaps[w->adj[ii].b_iid] == 1) { // Only one overlap, can't indicate sub read! //if ((subreadFile) && (subreadFileVerbose)) // fprintf(subreadFile, "oneOverlap %u (%u-%u) %u (%u-%u) -- can't indicate subreads\n", // w->adj[ii].a_iid, w->adj[ii].aovlbgn, w->adj[ii].aovlend, w->adj[ii].b_iid, w->adj[ii].bovlbgn, w->adj[ii].bovlend); continue; } // We should never get more than two overlaps per read pair. if (numOlaps[w->adj[ii].b_iid] > 2) { fprintf(stderr, "ERROR: more overlaps than expected for pair %u %u.\n", w->adj[ii].a_iid, w->adj[ii].b_iid); continue; } assert(numOlaps[w->adj[ii].b_iid] == 2); uint32 jj = secondIdx[w->adj[ii].b_iid]; adjOverlap *ajj = w->adj + jj; assert(jj < w->adjLen); if (ii == jj) { // Already did this one! //if ((subreadFile) && (subreadFileVerbose)) // fprintf(subreadFile, "sameOverlap %u (%u-%u) %u (%u-%u)\n", // w->adj[ii].a_iid, w->adj[ii].aovlbgn, w->adj[ii].aovlend, w->adj[ii].b_iid, w->adj[ii].bovlbgn, w->adj[ii].bovlend); continue; } // The two overlaps should be for the same reads. assert(w->adj[ii].a_iid == w->adj[jj].a_iid); assert(w->adj[ii].b_iid == w->adj[jj].b_iid); // And opposite orientations. if (w->adj[ii].flipped == w->adj[jj].flipped) { fprintf(stderr, "ERROR: same orient duplicate overlaps for pair %u %u\n", w->adj[ii].a_iid, w->adj[ii].b_iid); continue; } assert(w->adj[ii].flipped != w->adj[jj].flipped); bool AcheckSub = (doCheckSubRead(gkp, w->adj[ii].a_iid) == true); bool BcheckSub = (doCheckSubRead(gkp, w->adj[ii].b_iid) == true); assert(AcheckSub == true); // Otherwise we wouldn't be in this function! // Decide what type of duplicate we have. // Overlap on the A read -=> B read is potentially sub read containing -=> don't use overlaps // Overlap on the B read -=> A read is potentially sub read containing -=> split this read uint32 Aoverlap = intervalOverlap(w->adj[ii].aovlbgn, w->adj[ii].aovlend, w->adj[jj].aovlbgn, w->adj[jj].aovlend); uint32 Boverlap = intervalOverlap(w->adj[ii].bovlbgn, w->adj[ii].bovlend, w->adj[jj].bovlbgn, w->adj[jj].bovlend); // If there is no overlap anywhere, we're not sure what is going on. This could be a genomic // repeat. Leave the overlaps alone. // if ((Aoverlap == 0) && (Boverlap == 0)) continue; // Remember if the overlapping ovelap is large - we'll later check if the bad region falls // within here, and if there are enough spanning reads not trim. We also use this as one more // count of BAD. // if ((AcheckSub) && (Aoverlap > 1000) && (BcheckSub) && (Boverlap > 1000)) { uint32 dist = (w->adj[ii].a_iid > w->adj[ii].b_iid) ? (w->adj[ii].a_iid - w->adj[ii].b_iid) : (w->adj[ii].b_iid - w->adj[ii].a_iid); if (subreadFile) fprintf(subreadFile, " II %8u (%6u-%6u) %8u (%6u-%6u) JJ %8u (%6u-%6u) %8u (%6u-%6u) %s\n", w->adj[ii].a_iid, w->adj[ii].aovlbgn, w->adj[ii].aovlend, w->adj[ii].b_iid, w->adj[ii].bovlbgn, w->adj[ii].bovlend, w->adj[jj].a_iid, w->adj[jj].aovlbgn, w->adj[jj].aovlend, w->adj[jj].b_iid, w->adj[jj].bovlbgn, w->adj[jj].bovlend, (dist > 5) ? " PALINDROME WARNING--FAR-IID--WARNING" : "PALINDROME"); largePalindrome = true; } #if 0 // Otherwise, if the overlaps overlap on both reads by significant chunks, don't believe // either. These are possibly both chimeric reads, at least PacBio junction reads. // // Or an inverted repeat. // if ((AcheckSub) && (Aoverlap > 50) && (BcheckSub) && (Boverlap > 50)) { if (subreadFile) fprintf(subreadFile, "BothOv %u (%u-%u) %u (%u-%u) %u (%u-%u) %u (%u-%u)\n", w->adj[ii].a_iid, w->adj[ii].aovlbgn, w->adj[ii].aovlend, w->adj[ii].b_iid, w->adj[ii].bovlbgn, w->adj[ii].bovlend, w->adj[jj].a_iid, w->adj[jj].aovlbgn, w->adj[jj].aovlend, w->adj[jj].b_iid, w->adj[jj].bovlbgn, w->adj[jj].bovlend); } #endif #if 0 // Stronger overlap in the A reads. The B read looks like it has subreads, which is perfectly fine // evidence for us. Unless they span a junction. // if ((BcheckSub) && (Boverlap < Aoverlap)) { if (subreadFile) fprintf(subreadFile, "BcheckSub %u (%u-%u) %u (%u-%u) %u (%u-%u) %u (%u-%u)\n", w->adj[ii].a_iid, w->adj[ii].aovlbgn, w->adj[ii].aovlend, w->adj[ii].b_iid, w->adj[ii].bovlbgn, w->adj[ii].bovlend, w->adj[jj].a_iid, w->adj[jj].aovlbgn, w->adj[jj].aovlend, w->adj[jj].b_iid, w->adj[jj].bovlbgn, w->adj[jj].bovlend); } #endif // It looks like A has sub reads if the B read has a strong overlap in overlaps, and the A read does not // have a strong overlap. if ((Aoverlap > 250) || (Boverlap < 250)) // A strong overlap in the A read, there isn't a sub read junction we can identifiy, OR // A weak overlap in the B read, and we expected the B read to align to both of the A subreads. continue; // Decide on a region in the read that is suspected to contain the chimer junction. // // In the true case: ii overlap is first on the read; bad region from the end of this overlap // to the start of the jj overlap. // // Note that sometimes overlaps extend through the junction. This will just flip the region // around. We're expecting to find non-overlapping overlaps, but if we find overlapping ones, // the bad interval is still between the end points. // // --------------> ------------> // <--------- vs <--------- // uint32 badbgn = (w->adj[ii].aovlbgn < w->adj[jj].aovlbgn) ? w->adj[ii].aovlend : w->adj[jj].aovlend; uint32 badend = (w->adj[ii].aovlbgn < w->adj[jj].aovlbgn) ? w->adj[jj].aovlbgn : w->adj[ii].aovlbgn; if (badbgn > badend) { uint32 a = badbgn; badbgn = badend; badend = a; } assert(badbgn <= badend); if (subreadFile) fprintf(subreadFile, " II %8u (%6u-%6u) %8u (%6u-%6u) JJ %8u (%6u-%6u) %8u (%6u-%6u) BAD %6u-%6u size %6u %s\n", w->adj[ii].a_iid, w->adj[ii].aovlbgn, w->adj[ii].aovlend, w->adj[ii].b_iid, w->adj[ii].bovlbgn, w->adj[ii].bovlend, w->adj[jj].a_iid, w->adj[jj].aovlbgn, w->adj[jj].aovlend, w->adj[jj].b_iid, w->adj[jj].bovlbgn, w->adj[jj].bovlend, badbgn, badend, badend - badbgn, (badend - badbgn <= SUBREAD_LOOP_MAX_SIZE) ? "(EVIDENCE)" : "(too far)"); // A true subread signature will have a small bad interval (10 bases) and largely agree on the // interval. False signature will have a large size, and not agree. We only check for size // though. // if (badend - badbgn <= SUBREAD_LOOP_MAX_SIZE) BAD.add(badbgn, badend - badbgn); // Save all plausible pairs. // if (badend - badbgn <= SUBREAD_LOOP_EXT_SIZE) BADall.add(badbgn, badend - badbgn); } // // Merge all the 'bad' intervals. Save the merged intervals for later use. // BAD.merge(); BADall.merge(); for (uint32 bb=0; bbadjLen; ii++) if ((w->adj[ii].aovlbgn + 100 < BAD.lo(bb)) && (BAD.hi(bb) + 100 < w->adj[ii].aovlend)) numSpan += (doCheckSubRead(gkp, w->adj[ii].a_iid)) ? 1 : 2; if (subreadFile) fprintf(subreadFile, "AcheckSub region %u ("F_S32"-"F_S32") with %u hits %u bighits - span %u largePalindrome %s\n", w->adj[0].a_iid, BAD.lo(bb), BAD.hi(bb), BAD.count(bb), allHits, numSpan, largePalindrome ? "true" : "false"); if (numSpan > 9) // If there are 10 or more spanning read (equivalents) this is not a subread junction. There // is plenty of evidence it is true. continue; if (BAD.count(bb) + allHits / 4 + largePalindrome < 3) // If 2 or fewer reads claim this is a sub read junction, skip it. Evidence is weak. continue; if (subreadFile) fprintf(subreadFile, "CONFIRMED BAD REGION %d-%d\n", BAD.lo(bb), BAD.hi(bb)); w->blist.push_back(badRegion(w->id, badType_subread, BAD.lo(bb), BAD.hi(bb))); } } canu-1.3/src/overlapBasedTrimming/splitReads-trimBad.C000066400000000000000000000105731272335422300230170ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-JAN-19 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "splitReads.H" // If after chimer trimming a read had a bad interval in the clear, just delete the read. // Evidence said it was both good and bad. // // But first, if the bad interval just touches the clear, trim it out. // Process the list of bad intervals in blist to find a single clear range. // Store the result in w->clrBgn and w->clrEnd. void trimBadInterval(gkStore *gkp, workUnit *w, uint32 minReadLength, FILE *subreadFile, bool subreadFileVerbose) { if (w->blist.size() == 0) return; intervalList goodRegions; // Build an interval list of all the bad regions, and invert them into good regions. for (uint32 bb=0; bbblist.size(); bb++) goodRegions.add(w->blist[bb].bgn, w->blist[bb].end - w->blist[bb].bgn); goodRegions.invert(w->clrBgn, w->clrEnd); // Find the largest good region, save it in the output clear range. If there are no // regions (the whole read was marked bad?), default to a bougs clear range. w->clrBgn = UINT32_MAX; w->clrEnd = UINT32_MAX; for (uint32 rr=0; rrclrEnd - w->clrBgn) < (goodRegions.hi(rr) - goodRegions.lo(rr))) { w->clrBgn = goodRegions.lo(rr); w->clrEnd = goodRegions.hi(rr); } } // If the largest isn't big enough, remember that, and annotate the log appropriately if (w->clrEnd - w->clrBgn < minReadLength) w->isOK = false; // For logging, find the two bordering bad regions if (subreadFile) { vector loBad; vector hiBad; uint32 spur5 = UINT32_MAX; uint32 spur3 = UINT32_MAX; for (uint32 rr=0; rrblist.size(); rr++) { if (w->blist[rr].type == badType_5spur) { assert(spur5 == UINT32_MAX); // Should be at most one 5' spur region spur5 = rr; } if (w->blist[rr].type == badType_3spur) { assert(spur3 == UINT32_MAX); // Should be at most one 3' spur region spur3 = rr; } if (w->blist[rr].end == w->clrBgn) loBad.push_back(rr); if (w->clrEnd == w->blist[rr].bgn) hiBad.push_back(rr); } // Can't really say much about the number of regions we find. There could be none (if the // biggest good region extends to the end of the read), exactly one (what we expect), or more // than one (if two algorithms agree on a bad region). char *logPtr = w->logMsg; sprintf(logPtr, "iid %6u trim %7u %7u", w->id, w->clrBgn, w->clrEnd); while (*logPtr) logPtr++; if (w->isOK == false) sprintf(logPtr, " TOO_SHORT"); while (*logPtr) logPtr++; if (spur5 != UINT32_MAX) sprintf(logPtr, " (5'spur %7u %7u)", w->blist[spur5].bgn, w->blist[spur5].end); while (*logPtr) logPtr++; if (spur3 != UINT32_MAX) sprintf(logPtr, " (3'spur %7u %7u)", w->blist[spur3].bgn, w->blist[spur3].end); while (*logPtr) logPtr++; for (uint32 xx=0; xxblist[x].typeName(), w->blist[x].bgn, w->blist[x].end); while (*logPtr) logPtr++; } for (uint32 xx=0; xxblist[x].typeName(), w->blist[x].bgn, w->blist[x].end); while (*logPtr) logPtr++; } logPtr[0] = '\n'; logPtr[1] = 0; } } canu-1.3/src/overlapBasedTrimming/splitReads-workUnit.C000066400000000000000000000114051272335422300232520ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-APR-15 to 2015-JUN-16 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "splitReads.H" // Adjust the overlap for trimming already done. Filter out useless overlaps. Save the // overlap into a useful format. void workUnit::addAndFilterOverlaps(gkStore *gkp, clearRangeFile *finClr, double errorRate, ovOverlap *ovl, uint32 ovlLen) { if (adjMax < ovlLen) { delete [] adj; adjMax = ovlLen; adj = new adjOverlap [adjMax]; } adjLen = 0; for (uint32 oo=0; ooa_iid; int32 idB = o->b_iid; if (finClr->isDeleted(idA) || finClr->isDeleted(idB)) continue; // Returns the finClr clear range of the two reads (in *clr*), and the overlap adjusted to // that clear range (in *ovl*). The B read coordinates are reverse-complemented if the // overlap is reversed (so bovlbgn < bovlend always). a->a_iid = o->a_iid; a->b_iid = o->b_iid; a->flipped = o->flipped(); bool valid = false; if (o->flipped() == false) valid = adjustNormal(finClr, gkp, o, a->aovlbgn, a->aovlend, a->bovlbgn, a->bovlend, a->aclrbgn, a->aclrend, a->bclrbgn, a->bclrend); else valid = adjustFlipped(finClr, gkp, o, a->aovlbgn, a->aovlend, a->bovlbgn, a->bovlend, a->aclrbgn, a->aclrend, a->bclrbgn, a->bclrend); if (valid == false) // adjust() says the overlap doesn't intersect the clear range, so nothing here. continue; assert(a->aclrbgn <= a->aovlbgn); // Also checked in adjust()... assert(a->bclrbgn <= a->bovlbgn); assert(a->aovlend <= a->aclrend); assert(a->bovlend <= a->bclrend); // Reset evidence hangs that are close to zero to be zero. This shifts the overlap end point // to remove as much of the useless hang as possible. if (a->bovlbgn - a->bclrbgn < 15) { uint32 limit = a->aovlbgn - a->aclrbgn; uint32 adjust = a->bovlbgn - a->bclrbgn; if (adjust > limit) adjust = limit; if ((a->aovlbgn < adjust) || (a->bovlbgn < adjust)) fprintf(stderr, "ovl %u %u-%u %u %u-%u -> clr %u-%u %u-%u adj %u-%u %u-%u\n", o->a_iid, o->a_bgn(), o->a_end(), o->b_iid, o->b_bgn(), o->b_end(), a->aclrbgn, a->aclrend, a->bclrbgn, a->bclrend, a->aovlbgn, a->aovlend, a->bovlbgn, a->bovlend); assert(a->aovlbgn >= adjust); assert(a->bovlbgn >= adjust); a->aovlbgn -= adjust; a->bovlbgn -= adjust; assert(a->aclrbgn <= a->aovlbgn); assert(a->bclrbgn <= a->bovlbgn); } if (a->bclrend - a->bovlend < 15) { uint32 limit = a->aclrend - a->aovlend; uint32 adjust = a->bclrend - a->bovlend; if (adjust > limit) adjust = limit; a->aovlend += adjust; a->bovlend += adjust; assert(a->aovlend <= a->aclrend); assert(a->bovlend <= a->bclrend); } // Filter out garbage overlaps // // The first version used hard and fast cutoffs of (>=35bp and <= 0.02 error) or (>= 70bp). // These were not fair to short reads. // // The second version used compilacted rules (see cvs), but based the decision off of the // length of the A-read. Fair, but only for assemblies of similarily sized reads. Totally // unfair for Sanger-vs-Illumin overlaps. // // The third version sets a minimum length and identity based on the shorter fragment. These // mirror closely what the second version was doing. It was extended to allow even shorter // lengths if either read is aligned to an end. // // The fourth version, for long noisy reads, accepts all overlaps. if (0) continue; // Save the overlap. adjLen++; } } canu-1.3/src/overlapBasedTrimming/splitReads.C000066400000000000000000000435171272335422300214430ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-APR-15 to 2015-JUN-25 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "splitReads.H" #include "trimStat.H" #include "clearRangeFile.H" #include "AS_UTL_decodeRange.H" int main(int argc, char **argv) { char *gkpName = NULL; char *ovsName = NULL; char *finClrName = NULL; char *outClrName = NULL; double errorRate = 0.06; //uint32 minAlignLength = 40; uint32 minReadLength = 64; uint32 idMin = 1; uint32 idMax = UINT32_MAX; char *outputPrefix = NULL; char outputName[FILENAME_MAX]; FILE *staFile = NULL; FILE *reportFile = NULL; FILE *subreadFile = NULL; bool doSubreadLogging = true; bool doSubreadLoggingVerbose = false; // Statistics on the trimming - the second set are from the old logging, and don't really apply anymore. trimStat readsIn; // Read is eligible for trimming trimStat deletedIn; // Read was deleted already trimStat noTrimIn; // Read not requesting trimming trimStat noOverlaps; // no overlaps in store trimStat noCoverage; // no coverage after adjusting for trimming done trimStat readsProcChimera; // Read was processed for chimera signal trimStat readsProcSpur; // Read was processed for spur signal trimStat readsProcSubRead; // Read was processed for subread signal #if 0 trimStat badSpur5; trimStat badSpur3; trimStat badChimera; trimStat badSubread; #endif trimStat readsNoChange; trimStat readsBadSpur5, basesBadSpur5; trimStat readsBadSpur3, basesBadSpur3; trimStat readsBadChimera, basesBadChimera; trimStat readsBadSubread, basesBadSubread; trimStat readsTrimmed5; trimStat readsTrimmed3; #if 0 trimStat fullCoverage; // fully covered by overlaps trimStat noSignalNoGap; // no signal, no gaps trimStat noSignalButGap; // no signal, with gaps trimStat bothFixed; // both chimera and spur signal trimmed trimStat chimeraFixed; // only chimera signal trimmed trimStat spurFixed; // only spur signal trimmed trimStat bothDeletedSmall; // deleted because of both cimera and spur signals trimStat chimeraDeletedSmall; // deleted because of chimera signal trimStat spurDeletedSmall; // deleted because of spur signal trimStat spurDetectedNormal; // normal spur detected trimStat spurDetectedLinker; // linker spur detected trimStat chimeraDetectedInnie; // innpue-pair chimera detected trimStat chimeraDetectedOverhang; // overhanging chimera detected trimStat chimeraDetectedGap; // gap chimera detected trimStat chimeraDetectedLinker; // linker chimera detected #endif trimStat deletedOut; // Read was deleted by trimming argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { ovsName = argv[++arg]; } else if (strcmp(argv[arg], "-o") == 0) { outputPrefix = argv[++arg]; } else if (strcmp(argv[arg], "-t") == 0) { AS_UTL_decodeRange(argv[++arg], idMin, idMax); } else if (strcmp(argv[arg], "-Ci") == 0) { finClrName = argv[++arg]; } else if (strcmp(argv[arg], "-Co") == 0) { outClrName = argv[++arg]; } else if (strcmp(argv[arg], "-e") == 0) { errorRate = atof(argv[++arg]); //} else if (strcmp(argv[arg], "-l") == 0) { // minAlignLength = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-minlength") == 0) { minReadLength = atoi(argv[++arg]); } else { fprintf(stderr, "%s: unknown option '%s'\n", argv[0], argv[arg]); err++; } arg++; } if (errorRate < 0.0) err++; if ((gkpName == 0L) || (ovsName == 0L) || (outputPrefix == NULL) || (err)) { fprintf(stderr, "usage: %s -G gkpStore -O ovlStore -Ci input.clearFile -Co output.clearFile -o outputPrefix]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " -G gkpStore path to read store\n"); fprintf(stderr, " -O ovlStore path to overlap store\n"); fprintf(stderr, "\n"); fprintf(stderr, " -o name output prefix, for logging\n"); fprintf(stderr, "\n"); fprintf(stderr, " -t bgn-end limit processing to only reads from bgn to end (inclusive)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -Ci clearFile path to input clear ranges (NOT SUPPORTED)\n"); fprintf(stderr, " -Co clearFile path to ouput clear ranges\n"); fprintf(stderr, "\n"); fprintf(stderr, " -e erate ignore overlaps with more than 'erate' percent error\n"); //fprintf(stderr, " -l length ignore overlaps shorter than 'l' aligned bases (NOT SUPPORTED)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -minlength l reads trimmed below this many bases are deleted\n"); fprintf(stderr, "\n"); if (errorRate < 0.0) fprintf(stderr, "ERROR: Error rate (-e) value %f too small; must be 'fraction error' and above 0.0\n", errorRate); exit(1); } gkStore *gkp = gkStore::gkStore_open(gkpName); ovStore *ovs = new ovStore(ovsName, gkp); clearRangeFile *finClr = new clearRangeFile(finClrName, gkp); clearRangeFile *outClr = new clearRangeFile(outClrName, gkp); if (outClr) // If the outClr file exists, those clear ranges are loaded. We need to reset them // back to 'untrimmed' for now. outClr->reset(gkp); if (finClr && outClr) // A finClr file was supplied, so use those as the clear ranges. outClr->copy(finClr); sprintf(outputName, "%s.log", outputPrefix); errno = 0; reportFile = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", outputName, strerror(errno)), exit(1); sprintf(outputName, "%s.subread.log", outputPrefix); errno = 0; subreadFile = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", outputName, strerror(errno)), exit(1); uint32 ovlLen = 0; uint32 ovlMax = 64 * 1024; ovOverlap *ovl = ovOverlap::allocateOverlaps(gkp, ovlMax); memset(ovl, 0, sizeof(ovOverlap) * ovlMax); workUnit *w = new workUnit; if (idMin < 1) idMin = 1; if (idMax > gkp->gkStore_getNumReads()) idMax = gkp->gkStore_getNumReads(); fprintf(stderr, "Processing from ID "F_U32" to "F_U32" out of "F_U32" reads, using errorRate = %.2f\n", idMin, idMax, gkp->gkStore_getNumReads(), errorRate); for (uint32 id=idMin; id<=idMax; id++) { gkRead *read = gkp->gkStore_getRead(id); gkLibrary *libr = gkp->gkStore_getLibrary(read->gkRead_libraryID()); if (finClr->isDeleted(id)) { // Read already trashed. deletedIn += read->gkRead_sequenceLength(); continue; } if ((libr->gkLibrary_removeSpurReads() == false) && (libr->gkLibrary_removeChimericReads() == false) && (libr->gkLibrary_checkForSubReads() == false)) { // Nothing to do. noTrimIn += read->gkRead_sequenceLength(); continue; } readsIn += read->gkRead_sequenceLength(); uint32 nLoaded = ovs->readOverlaps(id, ovl, ovlLen, ovlMax); //fprintf(stderr, "read %7u with %7u overlaps\r", id, nLoaded); if (nLoaded == 0) { // No overlaps, nothing to check! noOverlaps += read->gkRead_sequenceLength(); continue; } w->clear(id, finClr->bgn(id), finClr->end(id)); w->addAndFilterOverlaps(gkp, finClr, errorRate, ovl, ovlLen); if (w->adjLen == 0) { // All overlaps trimmed out! noCoverage += read->gkRead_sequenceLength(); continue; } // Find bad regions. //if (libr->gkLibrary_markBad() == true) // // From an external file, a list of known bad regions. If no overlaps span // // the region with sufficient coverage, mark the region as bad. This was // // motivated by the old 454 linker detection. // markBad(gkp, w, subreadFile, doSubreadLoggingVerbose); //if (libr->gkLibrary_removeSpurReads() == true) { // readsProcSpur += read->gkRead_sequenceLength(); // detectSpur(gkp, w, subreadFile, doSubreadLoggingVerbose); // Get stats on spur region detected - save the length of each region to the trimStats object. //} //if (libr->gkLibrary_removeChimericReads() == true) { // readsProcChimera += read->gkRead_sequenceLength(); // detectChimer(gkp, w, subreadFile, doSubreadLoggingVerbose); // Get stats on chimera region detected - save the length of each region to the trimStats object. //} if (libr->gkLibrary_checkForSubReads() == true) { readsProcSubRead += read->gkRead_sequenceLength(); detectSubReads(gkp, w, subreadFile, doSubreadLoggingVerbose); } // Get stats on the bad regions found. This kind of duplicates code in trimBadInterval(), but // I don't want to pass all the stats objects into there. if (w->blist.size() == 0) { readsNoChange += read->gkRead_sequenceLength(); } else { uint32 nSpur5 = 0, bSpur5 = 0; uint32 nSpur3 = 0, bSpur3 = 0; uint32 nChimera = 0, bChimera = 0; uint32 nSubread = 0, bSubread = 0; for (uint32 bb=0; bbblist.size(); bb++) { switch (w->blist[bb].type) { case badType_5spur: nSpur5 += 1; basesBadSpur5 += w->blist[bb].end - w->blist[bb].bgn; break; case badType_3spur: nSpur3 += 1; basesBadSpur3 += w->blist[bb].end - w->blist[bb].bgn; break; case badType_chimera: nChimera += 1; basesBadChimera += w->blist[bb].end - w->blist[bb].bgn; break; case badType_subread: nSubread += 1; basesBadSubread += w->blist[bb].end - w->blist[bb].bgn; break; default: break; } } if (nSpur5 > 0) readsBadSpur5 += nSpur5; if (nSpur3 > 0) readsBadSpur3 += nSpur3; if (nChimera > 0) readsBadChimera += nChimera; if (nSubread > 0) readsBadSubread += nSubread; } // Find solution. This coalesces the list (in 'w') of all the bad regions found, picks out the // largest good region, generates a log of the bad regions that support this decision, and sets // the trim points. trimBadInterval(gkp, w, minReadLength, subreadFile, doSubreadLoggingVerbose); // Log the solution. AS_UTL_safeWrite(reportFile, w->logMsg, "logMsg", sizeof(char), strlen(w->logMsg)); // Save the solution.... outClr->setbgn(w->id) = w->clrBgn; outClr->setend(w->id) = w->clrEnd; // And maybe delete the read. if (w->isOK == false) { deletedOut += read->gkRead_sequenceLength(); outClr->setDeleted(w->id); } // Update stats on what was trimmed. The asserts say the clear range didn't expand, and the if // tests if the clear range changed. assert(w->clrBgn >= w->iniBgn); assert(w->iniEnd >= w->clrEnd); if (w->clrBgn > w->iniBgn) readsTrimmed5 += w->clrBgn - w->iniBgn; if (w->iniEnd > w->clrEnd) readsTrimmed3 += w->iniEnd - w->clrEnd; } delete [] ovl; delete w; gkp->gkStore_close(); delete finClr; delete outClr; // Close log files if (reportFile) fclose(reportFile); if (subreadFile) fclose(subreadFile); // Write the summary if (outputPrefix) { sprintf(outputName, "%s.stats", outputPrefix); errno = 0; staFile = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", outputName, strerror(errno)); } if (staFile == NULL) staFile = stdout; // Would like to know number of subreads per read fprintf(staFile, "PARAMETERS:\n"); fprintf(staFile, "----------\n"); fprintf(staFile, "%7u (reads trimmed below this many bases are deleted)\n", minReadLength); fprintf(staFile, "%7.4f (use overlaps at or below this fraction error)\n", errorRate); //fprintf(staFile, "%7u (use only overlaps longer than this)\n", minAlignLength); // NOT SUPPORTED! fprintf(staFile, "INPUT READS:\n"); fprintf(staFile, "-----------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (reads processed)\n", readsIn.nReads, readsIn.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (reads not processed, previously deleted)\n", deletedIn.nReads, deletedIn.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (reads not processed, in a library where trimming isn't allowed)\n", noTrimIn.nReads, noTrimIn.nBases); fprintf(staFile, "\n"); fprintf(staFile, "PROCESSED:\n"); fprintf(staFile, "--------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (no overlaps)\n", noOverlaps.nReads, noOverlaps.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (no coverage after adjusting for trimming done already)\n", noCoverage.nReads, noCoverage.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (processed for chimera)\n", readsProcChimera.nReads, readsProcChimera.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (processed for spur)\n", readsProcSpur.nReads, readsProcSpur.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (processed for subreads)\n", readsProcSubRead.nReads, readsProcSubRead.nBases); fprintf(staFile, "\n"); fprintf(staFile, "READS WITH SIGNALS:\n"); fprintf(staFile, "------------------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" signals (number of 5' spur signal)\n", readsBadSpur5.nReads, readsBadSpur5.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" signals (number of 3' spur signal)\n", readsBadSpur3.nReads, readsBadSpur3.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" signals (number of chimera signal)\n", readsBadChimera.nReads, readsBadChimera.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" signals (number of subread signal)\n", readsBadSubread.nReads, readsBadSubread.nBases); fprintf(staFile, "\n"); fprintf(staFile, "SIGNALS:\n"); fprintf(staFile, "-------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (size of 5' spur signal)\n", basesBadSpur5.nReads, basesBadSpur5.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (size of 3' spur signal)\n", basesBadSpur3.nReads, basesBadSpur3.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (size of chimera signal)\n", basesBadChimera.nReads, basesBadChimera.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (size of subread signal)\n", basesBadSubread.nReads, basesBadSubread.nBases); fprintf(staFile, "\n"); fprintf(staFile, "TRIMMING:\n"); fprintf(staFile, "--------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (trimmed from the 5' end of the read)\n", readsTrimmed5.nReads, readsTrimmed5.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (trimmed from the 3' end of the read)\n", readsTrimmed3.nReads, readsTrimmed3.nBases); #if 0 fprintf(staFile, "DELETED:\n"); fprintf(staFile, "-------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (deleted because of both cimera and spur signals)\n", bothDeletedSmall.nReads, bothDeletedSmall.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (deleted because of chimera signal)\n", chimeraDeletedSmall.nReads, chimeraDeletedSmall.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (deleted because of spur signal)\n", spurDeletedSmall.nReads, spurDeletedSmall.nBases); fprintf(staFile, "\n"); fprintf(staFile, "SPUR TYPES:\n"); fprintf(staFile, "----------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (normal spur detected)\n", spurDetectedNormal.nReads, spurDetectedNormal.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (linker spur detected)\n", spurDetectedLinker.nReads, spurDetectedLinker.nBases); fprintf(staFile, "\n"); fprintf(staFile, "CHIMERA TYPES:\n"); fprintf(staFile, "-------------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (innie-pair chimera detected)\n", chimeraDetectedInnie.nReads, chimeraDetectedInnie.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (overhanging chimera detected)\n", chimeraDetectedOverhang.nReads, chimeraDetectedOverhang.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (gap chimera detected)\n", chimeraDetectedGap.nReads, chimeraDetectedGap.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (linker chimera detected)\n", chimeraDetectedLinker.nReads, chimeraDetectedLinker.nBases); #endif // INPUT READS = ACCEPTED + TRIMMED + DELETED // SPUR TYPE = TRIMMED and DELETED spur and both categories // CHIMERA TYPE = TRIMMED and DELETED chimera and both categories if (staFile != stdout) fclose(staFile); exit(0); } canu-1.3/src/overlapBasedTrimming/splitReads.H000066400000000000000000000132201272335422300214340ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-APR-20 to 2015-JUN-16 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef SPLIT_READS_H #define SPLIT_READS_H #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include "adjustOverlaps.H" #include "clearRangeFile.H" #include "intervalList.H" #include "AS_UTL_decodeRange.H" #include #include using namespace std; // But then use only overlaps larger than this for some of the more questionable overlaps. #define MIN_INTERVAL_OVERLAP 60 // Same orient overlaps closer than this are evidence of PacBio subreads. #define SUBREAD_LOOP_MAX_SIZE 500 #define SUBREAD_LOOP_EXT_SIZE 2000 // Original //#define SUBREAD_LOOP_MAX_SIZE 100 //#define SUBREAD_LOOP_EXT_SIZE 0 // WITH_REPORT_FULL will ALL overlap evidence. // REPORT_OVERLAPS will print the incoming overlaps in the log. // #undef WITH_REPORT_FULL #undef REPORT_OVERLAPS #undef DEBUG_ISLINKER #undef DEBUG_INTERVAL #undef DEBUG_FILTER // An overlap, adjusted for trimming done already. // class adjOverlap { public: uint32 a_iid; uint32 b_iid; uint32 flipped; // The b read is 3' to 5' uint32 aclrbgn, aclrend; // Clear range of the two reads uint32 bclrbgn, bclrend; // B read coords are adjusted for any flip uint32 aovlbgn, aovlend; // Overlap extent, adjusted for the clear range uint32 bovlbgn, bovlend; uint32 aOVLbgn, aOVLend; // Original unadjusted overlap uint32 bOVLbgn, bOVLend; }; const uint32 badType_nothing = 0; const uint32 badType_5spur = 1; const uint32 badType_3spur = 2; const uint32 badType_chimera = 3; const uint32 badType_subread = 4; // A region that is bad. // class badRegion { public: badRegion() { id = 0; type = 0; bgn = 0; end = 0; }; badRegion(uint32 id_, uint32 type_, uint32 bgn_, uint32 end_) { id = id_; type = type_; bgn = bgn_; end = end_; }; const char *typeName(void) const { const char *N[5] = { "nothing", "5'spur", "3'spur", "chimera", "subread" }; return(N[type]); } public: uint32 id; // Read ID for this bad region uint32 type; // Type of bad region - 5'spur, 3'spur, chimera, subread uint32 bgn; // uint32 end; // }; class workUnit { public: workUnit(uint32 id_=0, uint32 iniBgn_=0, uint32 iniEnd_=0) { clear(id_, iniBgn_, iniEnd_); logMsg[0] = 0; blist.clear(); adjLen = 0; adjMax = 0; adj = NULL; }; ~workUnit() { delete [] adj; }; void clear(uint32 id_, uint32 iniBgn_, uint32 iniEnd_) { id = id_; iniBgn = iniBgn_; iniEnd = iniEnd_; isOK = true; isBad = false; isSpur5 = false; isSpur3 = false; isChimera = false; isSubread = false; clrBgn = iniBgn_; clrEnd = iniEnd_; logMsg[0] = 0; blist.clear(); adjLen = 0; //adjMax = 0; // Do NOT reset the allocated space. //adj = NULL; }; void addAndFilterOverlaps(gkStore *gkp, clearRangeFile *finClr, double errorRate, ovOverlap *ovl, uint32 ovlLen); public: uint32 id; // Read ID uint32 iniBgn; // The input clear range uint32 iniEnd; // Results bool isOK; // Read is acceptable, might have been fixed as below bool isBad; // Read couldn't be fixed bool isSpur5; // Spur trimmed on 5' end bool isSpur3; // Spur trimmed on 3' end bool isChimera; // Read split because of suspected chimeric region bool isSubread; // Read split because of suspected subreads uint32 clrBgn; // The final clear range uint32 clrEnd; char logMsg[1024]; // Work space vector blist; // Overlaps uint32 adjLen; uint32 adjMax; adjOverlap *adj; }; void detectSpur(gkStore *gkp, workUnit *w, FILE *subreadFile, bool subreadFileVerbose); void detectChimer(gkStore *gkp, workUnit *w, FILE *subreadFile, bool subreadFileVerbose); void detectSubReads(gkStore *gkp, workUnit *w, FILE *subreadFile, bool subreadFileVerbose); void trimBadInterval(gkStore *gkp, workUnit *w, uint32 minReadLength, FILE *subreadFile, bool subreadFileVerbose); #endif // SPLIT_READS_H canu-1.3/src/overlapBasedTrimming/splitReads.mk000066400000000000000000000012051272335422300216540ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := splitReads SOURCES := splitReads.C \ splitReads-workUnit.C \ splitReads-subReads.C \ splitReads-trimBad.C \ adjustNormal.C \ adjustFlipped.C SRC_INCDIRS := .. ../AS_UTL ../stores TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/overlapBasedTrimming/test-random-chimeric-fragments.pl000066400000000000000000000074111272335422300255550ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## se strict; # ########################################################################### # # This file is part of Celera Assembler, a software program that # assembles whole-genome shotgun reads into contigs and scaffolds. # Copyright (C) 2009, J. Craig Venter Instititue. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received (LICENSE.txt) a copy of the GNU General Public # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # ########################################################################### # # Figures out what pieces ended up after trimming. Use with generate-random-chimeric-fragments.pl # # User unfriendly. my $uid; my $iid; open(F, "/work/wgs/FreeBSD-amd64/bin/gatekeeper -dumpfragments /work/test/test.gkpStore |") or die; while () { if (m/^fragmentIdent\s+=\s+(.*),(\d+)$/) { $uid = $1; $iid = $2; } if (m/^fragmentIsDeleted\s+=\s+1$/) { undef $uid; undef $iid; } if (m/^fragmentClear\s+=\s+OBTCHIMERA,(\d+),(\d+)$/) { my $bgn = $1; my $end = $2; my $b = 0; my $e = 0; if ($uid =~ m/(\d+)-/) { my @v = split '-', $uid; shift @v; # iid of read my $numMatches = 0; my $strMatches = ""; my $badMatches = 0; while (scalar(@v) > 0) { my $typ = shift @v; my $len = shift @v; $e = $b + $len; if (($bgn < $e - 5) && ($b + 5 < $end)) { $numMatches++; $strMatches .= "$b-$e-$typ "; if (($typ ne "seq") && ($typ ne "rev")) { $badMatches++; } } $b = $e; } if (($numMatches > 1) || ($badMatches > 0)) { my $ovl = `/work/wgs/FreeBSD-amd64/bin/overlapStore -g /work/test/test.gkpStore -dp $iid /work/test/0-overlaptrim/test.obtStore`; if (length($ovl) > 0) { print "$uid\n"; print "$bgn-$end\n"; print "$strMatches\n"; print $ovl; print "========================================================================================================================\n"; } } } } } close(F); canu-1.3/src/overlapBasedTrimming/trimReads-bestEdge.C000066400000000000000000000255251272335422300230020ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAY-28 to 2015-JUN-16 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "trimReads.H" #include #include #include using namespace std; // Generates plots for each trim. #undef GNUPLOT // Logging to the screen. #undef VERBOSE bool bestEdge(ovOverlap *ovl, uint32 ovlLen, gkRead *read, uint32 ibgn, uint32 iend, uint32 &fbgn, uint32 &fend, char *logMsg, uint32 errorValue, uint32 minOverlap, uint32 minCoverage, uint32 minReadLength) { fbgn = ibgn; fend = iend; logMsg[0] = 0; assert(read->gkRead_readID() == ovl[0].a_iid); assert(ovlLen > 0); // // Trim once, using the largest covered rule. This gets rid of any gaps in overlap coverage, // etc. // uint32 lbgn = 0; uint32 lend = 0; if (largestCovered(ovl, ovlLen, read, ibgn, iend, lbgn, lend, logMsg, errorValue, minOverlap, minCoverage, minReadLength) == false) return(false); // // Trim again, to maximize overlap length. // int32 iid = read->gkRead_readID(); uint32 len = read->gkRead_sequenceLength(); vector trim5; vector trim3; uint32 nContained = 0; #ifdef GNUPLOT vector trim5iid, trim5sco; vector trim3iid, trim3sco; #endif // For each overlap, add potential trim points where the overlap ends. for (uint32 i=0; i errorValue) // Overlap is crappy. continue; // Add trim points as long as they aren't outside the largest covered region. if (lbgn <= tbgn) trim5.push_back(tbgn); if (tend <= lend) trim3.push_back(tend); // If overlap indicates this read is contained, we're done. There isn't any // trimming needed. We can set the final trim and return, or let the rest // of the algorithm/logging finish. if ((tbgn == ibgn) && (tend == iend)) nContained++; } // If the read is contained in more than one other read, we're done. No trimming needed. if (nContained >= 2) { //fbgn = ibgn; //fend = iend; //return(true); trim5.clear(); trim3.clear(); } // Add trim points for the largest covered, i.e., no trimming past what largest covered did. trim5.push_back(lbgn); trim3.push_back(lend); // Duplicate removal, and possibly the processing algorithm, need the trim points // sorted from outside-in. sort(trim5.begin(), trim5.end(), std::less()); sort(trim3.begin(), trim3.end(), std::greater()); // Remove duplicate points (easier here than in the loops below) { uint32 old = 0; uint32 sav = 0; for (old=0, sav=0; old= len - triml) // Not possible to get a higher score by trimming more. break; //fprintf(stderr, "trim5 pt %u out of %u\n", pt, trim5.size()); for (uint32 i=0; i < ovlLen; i++) { uint32 tbgn = ibgn + ovl[i].a_bgn(); uint32 tend = ibgn + ovl[i].a_end(); if ((triml < tbgn) || (tend <= triml)) // Alignment starts after of the trim point; not a valid overlap. // or, alignment ends before the trim point (trimmed out). continue; // Limit the overlap to the largest covered. if (tend > lend) tend = lend; assert(tend >= triml); uint32 tlen = tend - triml; assert(tlen <= len); // Save the best score. Break ties by favoring the current best. if (((score < tlen)) || ((score == tlen) && (ovl[i].b_iid == best5iid))) { score = tlen; sciid = ovl[i].b_iid; scend = tend; #ifdef GNUPLOT trim5sco[pt] = score; trim5iid[pt] = sciid; #endif } } // Give up if we're not finding anything longer after 1/3 of the current read. Previous // attempts at this wanted to ensure that the current read (sciid) is the same as the best // (best5iid) but ties and slightly different begin/end points make this unreliable. For // example, an overlap from 100-500 and an overlap from 200-501. The first is longest up until // trim point 200, then the second becomes longest. The BEST longest is still the first // overlap, at trim point 100. if (score < best5score * 0.66) break; // Save a new best if the score is better, and the endpoint is more interior to the read. if ((best5score < score) && (best5end < scend)) { //fprintf(stderr, "RESET 5 end to pt %d score %d iid %d end %d\n", // pt, score, sciid, scend); best5pt = pt; best5score = score; best5iid = sciid; best5end = scend; } } //fprintf(stderr, "BEST at %u position %u pt %u\n", best5score, trim5[best5pt], best5pt); // Find the best 3' point. for (uint32 pt=0; pt= trimr - 0) // Not possible to get a higher score by trimming more. break; //fprintf(stderr, "trim3 pt %u out of %u\n", pt, trim3.size()); for (uint32 i=0; i < ovlLen; i++) { uint32 tbgn = ibgn + ovl[i].a_bgn(); uint32 tend = ibgn + ovl[i].a_end(); if ((tend < trimr) || (trimr <= tbgn)) // Alignment ends before the trim point; not a valid overlap, // or, alignment starts after the trim point (trimmed out) continue; // Limit the overlap to the largest covered. if (tbgn < lbgn) tbgn = lbgn; assert(trimr >= tbgn); uint32 tlen = trimr - tbgn; assert(tlen <= len); if (((score < tlen)) || ((score == tlen) && (ovl[i].b_iid == best3iid))) { score = tlen; sciid = ovl[i].b_iid; scbgn = tbgn; #ifdef GNUPLOT trim3sco[pt] = score; trim3iid[pt] = sciid; #endif } } if (score < best3score * 0.66) break; if ((best3score < score) && (scbgn < best3bgn)) { //fprintf(stderr, "RESET 3 end to pt %d score %d iid %d bgn %d\n", // pt, score, sciid, scbgn); best3pt = pt; best3score = score; best3iid = sciid; best3bgn = scbgn; } } #ifdef GNUPLOT { char D[FILENAME_MAX]; char G[FILENAME_MAX]; char S[FILENAME_MAX]; FILE *F; sprintf(D, "trim-%08d.dat", read->gkRead_readID()); sprintf(G, "trim-%08d.gp", read->gkRead_readID()); sprintf(S, "gnuplot < trim-%08d.gp", read->gkRead_readID()); F = fopen(D, "w"); for (uint32 i=0; igkRead_readIID()); fprintf(F, "plot \"trim-%08d.dat\" using 3:5 with linespoints, \"trim-%08d.dat\" using 10:12 with linespoints\n", read->gkRead_readIID(), read->gkRead_readIID()); fclose(F); system(S); } #endif //fprintf(stderr, "BEST at %u position %u pt %u\n", best3score, trim3[best3pt], best3pt); // Set trimming. Be just a little aggressive, and get rid of an extra base or two, if possible. // If not possible, the read is crap, and will be deleted anyway. fbgn = trim5[best5pt]; fend = trim3[best3pt]; if ((fbgn + 4 < fend) && (2 < fend)) { fbgn += 2; fend -= 2; } // Did we go outside the largestCovered region? Just reset. Ideally we'd assert, and crash. if (fbgn < lbgn) fbgn = lbgn; if (lend < fend) fend = lend; // Lastly, did we just end up with a bogus trim? There isn't any guard against picking the 5' // trim point to the right of the 3' trim point. #if 1 if (fend < fbgn) { fprintf(stderr, "iid = %u\n", read->gkRead_readID()); fbgn = lbgn; fend = lend; } #endif assert(fbgn <= fend); return(true); } canu-1.3/src/overlapBasedTrimming/trimReads-largestCovered.C000066400000000000000000000147661272335422300242360ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAY-28 to 2015-JUN-16 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "trimReads.H" #include "intervalList.H" bool largestCovered(ovOverlap *ovl, uint32 ovlLen, gkRead *read, uint32 ibgn, uint32 iend, uint32 &fbgn, uint32 &fend, char *logMsg, uint32 errorValue, uint32 minOverlap, uint32 minCoverage, uint32 minReadLength) { logMsg[0] = 0; assert(read->gkRead_readID() == ovl[0].a_iid); assert(ovlLen > 0); intervalList IL; intervalList ID; int32 iid = read->gkRead_readID(); uint32 nSkip = 0; uint32 nUsed = 0; for (uint32 i=0; i errorValue) { // Overlap is crappy. //fprintf(stderr, "skip %2u\n", i); nSkip++; continue; } //fprintf(stderr, "save %2u\n", i); nUsed++; IL.add(tbgn, tend - tbgn); } #if 0 for (uint32 it=0; it 0) { intervalList DE(IL); uint32 it = 0; uint32 ib = 0; uint32 ie = 0; while (it < DE.numberOfIntervals()) { //fprintf(stderr, "DE - %d - "F_S64" "F_S64" "F_U32"\n", fr.gkFragment_getReadIID(), DE.lo(it), DE.hi(it), DE.depth(it)); if (DE.depth(it) < minCoverage) { // Dropped below good coverage depth. If we have an interval, save it. Reset. if (ie > ib) { //fprintf(stderr, "AD1 %d-%d len %d\n", ib, ie, ie - ib); ID.add(ib, ie - ib); } ib = 0; ie = 0; } else if ((ib == 0) && (ie == 0)) { // Depth is good. If no current interval, make a new one. ib = DE.lo(it); ie = DE.hi(it); //fprintf(stderr, "NE1 %d-%d len %d\n", ib, ie, ie - ib); } else if (ie == DE.lo(it)) { // Depth is good. If this interval is adjacent to the current, extend. ie = DE.hi(it); //fprintf(stderr, "EXT %d-%d len %d\n", ib, ie, ie - ib); } else { // Depth is good, but we just had a gap in coverage. Save any current interval. Reset. if (ie > ib) { //fprintf(stderr, "AD2 %d-%d len %d\n", ib, ie, ie - ib); ID.add(ib, ie - ib); } ib = DE.lo(it); ie = DE.hi(it); //fprintf(stderr, "NE2 %d-%d len %d\n", ib, ie, ie - ib); } it++; } if (ie > ib) { //fprintf(stderr, "AD3 %d-%d len %d\n", ib, ie, ie - ib); ID.add(ib, ie - ib); } } // Now that we've created depth, merge the intervals. IL.merge(minOverlap); // IL - covered interavls enforcing a minimum overlap size (these can overlap) // ID - covered intervals enforcing a minimum depth (these cannot overlap) // // Create new intervals from the intersection of IL and ID. // // This catches one nasty case, where a thin overlap has more than minDepth coverage. // // ------------- 3x coverage // ------------- all overlaps 1 or 2 dashes long // --------- // ----------- if (minCoverage > 0) { intervalList FI; uint32 li = 0; uint32 di = 0; while ((li < IL.numberOfIntervals()) && (di < ID.numberOfIntervals())) { uint32 ll = IL.lo(li); uint32 lh = IL.hi(li); uint32 dl = ID.lo(di); uint32 dh = ID.hi(di); uint32 nl = 0; uint32 nh = 0; // If they intersect, make a new region if ((ll <= dl) && (dl < lh)) { nl = dl; nh = (lh < dh) ? lh : dh; } if ((dl <= ll) && (ll < dh)) { nl = ll; nh = (lh < dh) ? lh : dh; } if (nl < nh) FI.add(nl, nh - nl); // Advance the list with the earlier region. if (lh <= dh) // IL ends at or before ID li++; if (dh <= lh) { // ID ends at or before IL di++; } } // Replace the intervals to use with the intersection. IL = FI; } //////////////////////////////////////// // The IL.ct(it) is always 1 if we filter low coverage. It is no longer reported. #if 0 if (IL.numberOfIntervals() > 1) for (uint32 it=0; itgkRead_readID(), IL.lo(it), IL.hi(it)); #endif if (IL.numberOfIntervals() == 0) { strcpy(logMsg, "\tno high quality overlaps"); return(false); } fbgn = IL.lo(0); fend = IL.hi(0); sprintf(logMsg, "\tskipped %u overlaps; used %u overlaps", nSkip, nUsed); for (uint32 it=0; it fend - fbgn) { fbgn = IL.lo(it); fend = IL.hi(it); } } return(true); } canu-1.3/src/overlapBasedTrimming/trimReads-quality.C000066400000000000000000000143031272335422300227400ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2015-NOV-23 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "trimReads.H" // This is mostly historical. // // It needs to be updated to use sanger qv encoding, and to take that as input. // A simple initialized array -- performs a quality letter -> quality // value translation. // class qualityLookup { public: qualityLookup() { for (uint32 i=0; i<255; i++) q[i] = 1 / pow(10, i / 10.0); }; ~qualityLookup() { }; double lookup(uint32 x) { return(q[x]); }; private: double q[255]; }; qualityLookup qual; static void findGoodQuality(double *qltD, uint32 qltLen, char minQualityLetter, uint32 &qltL, uint32 &qltR) { struct pair { uint32 start; uint32 end; }; pair *f = new pair [qltLen + 1]; pair *r = new pair [qltLen + 1]; uint32 fpos=0, flen=0; uint32 rpos=0, rlen=0; uint32 p = 0; double q = 0; double minQuality = qual.lookup(minQualityLetter - '!'); // Scan forward, find first base with quality >= 20. // Then, find the first base that makes cumulative expected #errors > 1/100 // #errors = 1 / 10 ^ log(q) // while (p < qltLen) { // Find the next begin point // while ((p < qltLen) && (qltD[p] > minQuality)) p++; // Got a begin point! Scan until the quality drops significantly. // f[fpos].start = p; f[fpos].end = p; q = qltD[p]; p++; while ((p < qltLen) && (q / (p - f[fpos].start) < minQuality)) { q += qltD[p]; p++; } f[fpos].end = p; if (f[fpos].end - f[fpos].start > 10) fpos++; } // Scan backward, just like the forward. // // Stung by using uint32 for p; p is one more than it wants to be. // Although, to be fair, there are just about as many cases of p+1 // as p below. // p = qltLen; q = 0; while (p > 0) { while ((p > 0) && (qltD[p-1] > minQuality)) p--; r[rpos].start = p; r[rpos].end = p; if (p > 0) { p--; q = qltD[p]; while ((p > 0) && (q / (r[rpos].end - p) < minQuality)) { p--; q += qltD[p]; } r[rpos].start = p; if (r[rpos].end - r[rpos].start > 10) rpos++; } } // Now, just pick the largest overlap qltL = 0; qltR = 0; flen = fpos; rlen = rpos; //fprintf(stderr, "qltLen = "F_U32" flen="F_U32" rlen="F_U32"\n", qltLen, flen, rlen); uint32 winningFPos = 0; uint32 winningRPos = 0; uint32 winningStyle = 0; for (fpos=0; fpos (qltR - qltL)) { winningFPos = fpos; winningRPos = rpos; winningStyle = 0; qltL = f[fpos].start; qltR = r[rpos].end; } } // fffffffffff // rrrrrrrrrr // else if ((f[fpos].start <= r[rpos].start) && (r[rpos].start <= f[fpos].end) && (f[fpos].end <= r[rpos].end)) { if ((f[fpos].end - r[rpos].start) > (qltR - qltL)) { winningFPos = fpos; winningRPos = rpos; winningStyle = 1; qltL = r[rpos].start; qltR = f[fpos].end; } } // fffffffffffffffffff // rrrrrrrrrr // else if ((f[fpos].start <= r[rpos].start) && (r[rpos].end <= f[fpos].end)) { if ((r[rpos].end - r[rpos].start) > (qltR - qltL)) { winningFPos = fpos; winningRPos = rpos; winningStyle = 2; qltL = r[rpos].start; qltR = r[rpos].end; } } // fffffffffff // rrrrrrrrrrrrrrrrrrrr // else if ((r[rpos].start <= f[fpos].start) && (f[fpos].end <= r[rpos].end)) { if ((f[fpos].end - f[fpos].start) > (qltR - qltL)) { winningFPos = fpos; winningRPos = rpos; winningStyle = 3; qltL = f[fpos].start; qltR = f[fpos].end; } } else if (f[fpos].end < r[rpos].start) { // NOP, no overlap. } else if (r[rpos].end < f[fpos].start) { // NOP, no overlap. } else { fprintf(stderr, "UNMATCHED OVERLAP\t"F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\n", f[fpos].start, f[fpos].end, r[rpos].start, r[rpos].end); } } } delete [] f; delete [] r; } // Takes a gkFragment, returns clear ranges for some quality // threshold. Higher level than I wanted, but it obscures // everything, and is exactly the interface that this and // mergeTrimming.C want. // void doTrim(gkRead *read, gkReadData *readData, double minQuality, uint32 &left, uint32 &right) { uint32 qltLen = read->gkRead_sequenceLength(); char *qltC = readData->gkReadData_getQualities(); double *qltD = new double [qltLen]; for (uint32 i=0; i end) // the original clear range was completely outside the max range. // bool enforceMaximumClearRange(gkRead *read, uint32 UNUSED(ibgn), uint32 UNUSED(iend), uint32 &fbgn, uint32 &fend, char *logMsg, clearRangeFile *maxClr) { if (maxClr == NULL) return(true); if (fbgn == fend) return(true); uint32 mbgn = maxClr->bgn(read->gkRead_readID()); uint32 mend = maxClr->end(read->gkRead_readID()); assert(mbgn < mend); assert(fbgn <= fend); if ((fend < mbgn) || (mend < fbgn)) { // Final clear not intersecting maximum clear. strcat(logMsg, (logMsg[0]) ? " - " : "\t"); strcat(logMsg, "outside maximum allowed clear range"); return(false); } else if ((fbgn < mbgn) || (mend < fend)) { // Final clear extends outside the maximum clear. fbgn = MAX(fbgn, mbgn); fend = MIN(fend, mend); strcat(logMsg, (logMsg[0]) ? " - " : "\t"); strcat(logMsg, "adjusted to obey maximum allowed clear range"); return(true); } else { // Final clear already within the maximum clear. return(true); } } int main(int argc, char **argv) { char *gkpName = 0L; char *ovsName = 0L; char *iniClrName = NULL; char *maxClrName = NULL; char *outClrName = NULL; uint32 errorValue = AS_OVS_encodeEvalue(0.015); uint32 minAlignLength = 40; uint32 minReadLength = 64; char *outputPrefix = NULL; char logName[FILENAME_MAX] = {0}; char sumName[FILENAME_MAX] = {0}; FILE *logFile = 0L; FILE *staFile = 0L; uint32 idMin = 1; uint32 idMax = UINT32_MAX; uint32 minEvidenceOverlap = 40; uint32 minEvidenceCoverage = 1; // Statistics on the trimming trimStat readsIn; // Read is eligible for trimming trimStat deletedIn; // Read was deleted already trimStat noTrimIn; // Read not requesting trimming trimStat readsOut; // Read was trimmed to a valid read trimStat noOvlOut; // Read was deleted; no ovelaps trimStat deletedOut; // Read was deleted; too small after trimming trimStat noChangeOut; // Read was untrimmed trimStat trim5; // Bases trimmed from the 5' end trimStat trim3; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { ovsName = argv[++arg]; } else if (strcmp(argv[arg], "-Ci") == 0) { iniClrName = argv[++arg]; } else if (strcmp(argv[arg], "-Cm") == 0) { maxClrName = argv[++arg]; } else if (strcmp(argv[arg], "-Co") == 0) { outClrName = argv[++arg]; } else if (strcmp(argv[arg], "-e") == 0) { double erate = atof(argv[++arg]); errorValue = AS_OVS_encodeEvalue(erate); } else if (strcmp(argv[arg], "-l") == 0) { minAlignLength = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-minlength") == 0) { minReadLength = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-ol") == 0) { minEvidenceOverlap = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-oc") == 0) { minEvidenceCoverage = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-o") == 0) { outputPrefix = argv[++arg]; } else if (strcmp(argv[arg], "-t") == 0) { AS_UTL_decodeRange(argv[++arg], idMin, idMax); } else { fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); err++; } arg++; } if ((gkpName == NULL) || (ovsName == NULL) || (outputPrefix == NULL) || (err)) { fprintf(stderr, "usage: %s -G gkpStore -O ovlStore -Co output.clearFile -o outputPrefix\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " -G gkpStore path to read store\n"); fprintf(stderr, " -O ovlStore path to overlap store\n"); fprintf(stderr, "\n"); fprintf(stderr, " -o name output prefix, for logging\n"); fprintf(stderr, "\n"); fprintf(stderr, " -t bgn-end limit processing to only reads from bgn to end (inclusive)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -Ci clearFile path to input clear ranges (NOT SUPPORTED)\n"); //fprintf(stderr, " -Cm clearFile path to maximal clear ranges\n"); fprintf(stderr, " -Co clearFile path to ouput clear ranges\n"); fprintf(stderr, "\n"); fprintf(stderr, " -e erate ignore overlaps with more than 'erate' percent error\n"); //fprintf(stderr, " -l length ignore overlaps shorter than 'l' aligned bases (NOT SUPPORTED)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -ol l the minimum evidence overlap length\n"); fprintf(stderr, " -oc c the minimum evidence overlap coverage\n"); fprintf(stderr, " evidence overlaps must overlap by 'l' bases to be joined, and\n"); fprintf(stderr, " must be at least 'c' deep to be retained\n"); fprintf(stderr, "\n"); fprintf(stderr, " -minlength l reads trimmed below this many bases are deleted\n"); fprintf(stderr, "\n"); exit(1); } gkStore *gkp = gkStore::gkStore_open(gkpName); ovStore *ovs = new ovStore(ovsName, gkp); clearRangeFile *iniClr = (iniClrName == NULL) ? NULL : new clearRangeFile(iniClrName, gkp); clearRangeFile *maxClr = (maxClrName == NULL) ? NULL : new clearRangeFile(maxClrName, gkp); clearRangeFile *outClr = (outClrName == NULL) ? NULL : new clearRangeFile(outClrName, gkp); if (outClr) // If the outClr file exists, those clear ranges are loaded. We need to reset them // back to 'untrimmed' for now. outClr->reset(gkp); if (iniClr && outClr) // An iniClr file was supplied, so use those as the initial clear ranges. outClr->copy(iniClr); if (outputPrefix) { sprintf(logName, "%s.log", outputPrefix); errno = 0; logFile = fopen(logName, "w"); if (errno) fprintf(stderr, "Failed to open log file '%s' for writing: %s\n", logName, strerror(errno)), exit(1); fprintf(logFile, "id\tinitL\tinitR\tfinalL\tfinalR\tmessage (DEL=deleted NOC=no change MOD=modified)\n"); } uint32 ovlLen = 0; uint32 ovlMax = 64 * 1024; ovOverlap *ovl = ovOverlap::allocateOverlaps(gkp, ovlMax); memset(ovl, 0, sizeof(ovOverlap) * ovlMax); char logMsg[1024] = {0}; if (idMin < 1) idMin = 1; if (idMax > gkp->gkStore_getNumReads()) idMax = gkp->gkStore_getNumReads(); fprintf(stderr, "Processing from ID "F_U32" to "F_U32" out of "F_U32" reads.\n", idMin, idMax, gkp->gkStore_getNumReads()); for (uint32 id=idMin; id<=idMax; id++) { gkRead *read = gkp->gkStore_getRead(id); gkLibrary *libr = gkp->gkStore_getLibrary(read->gkRead_libraryID()); logMsg[0] = 0; // If the fragment is deleted, do nothing. If the fragment was deleted AFTER overlaps were // generated, then the overlaps will be out of sync -- we'll get overlaps for these fragments // we skip. // if ((iniClr) && (iniClr->isDeleted(id) == true)) { deletedIn += read->gkRead_sequenceLength(); continue; } // If it did not request trimming, do nothing. Similar to the above, we'll get overlaps to // fragments we skip. // if ((libr->gkLibrary_finalTrim() == GK_FINALTRIM_LARGEST_COVERED) && (libr->gkLibrary_finalTrim() == GK_FINALTRIM_BEST_EDGE)) { noTrimIn += read->gkRead_sequenceLength(); continue; } readsIn += read->gkRead_sequenceLength(); // Decide on the initial trimming. We copied any iniClr into outClr above, and if there wasn't // an iniClr, then outClr is the full read. uint32 ibgn = outClr->bgn(id); uint32 iend = outClr->end(id); // Set the, ahem, initial final trimming. bool isGood = false; uint32 fbgn = ibgn; uint32 fend = iend; // Load overlaps. uint32 nLoaded = ovs->readOverlaps(id, ovl, ovlLen, ovlMax); // Trim! if (nLoaded == 0) { // No overlaps, so mark it as junk. isGood = false; } else if (libr->gkLibrary_finalTrim() == GK_FINALTRIM_LARGEST_COVERED) { // Use the largest region covered by overlaps as the trim assert(ovlLen > 0); assert(id == ovl[0].a_iid); isGood = largestCovered(ovl, ovlLen, read, ibgn, iend, fbgn, fend, logMsg, errorValue, minEvidenceOverlap, minEvidenceCoverage, minReadLength); assert(fbgn <= fend); } else if (libr->gkLibrary_finalTrim() == GK_FINALTRIM_BEST_EDGE) { // Use the largest region covered by overlaps as the trim assert(ovlLen > 0); assert(id == ovl[0].a_iid); isGood = bestEdge(ovl, ovlLen, read, ibgn, iend, fbgn, fend, logMsg, errorValue, minEvidenceOverlap, minEvidenceCoverage, minReadLength); assert(fbgn <= fend); } else { // Do nothing. Really shouldn't get here. assert(0); continue; } // Enforce the maximum clear range if ((isGood) && (maxClr)) { isGood = enforceMaximumClearRange(read, ibgn, iend, fbgn, fend, logMsg, maxClr); assert(fbgn <= fend); } // // Trimmed. Make sense of the result, write some logs, and update the output. // // If bad trimming or too small, write the log and keep going. // if (nLoaded == 0) { noOvlOut += read->gkRead_sequenceLength(); outClr->setbgn(id) = fbgn; outClr->setend(id) = fend; outClr->setDeleted(id); // Gah, just obliterates the clear range. fprintf(logFile, F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\tNOV%s\n", id, ibgn, iend, fbgn, fend, (logMsg[0] == 0) ? "" : logMsg); } else if ((isGood == false) || (fend - fbgn < minReadLength)) { deletedOut += read->gkRead_sequenceLength(); outClr->setbgn(id) = fbgn; outClr->setend(id) = fend; outClr->setDeleted(id); // Gah, just obliterates the clear range. fprintf(logFile, F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\tDEL%s\n", id, ibgn, iend, fbgn, fend, (logMsg[0] == 0) ? "" : logMsg); } // If we didn't change anything, also write a log. // else if ((ibgn == fbgn) && (iend == fend)) { noChangeOut += read->gkRead_sequenceLength(); fprintf(logFile, F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\tNOC%s\n", id, ibgn, iend, fbgn, fend, (logMsg[0] == 0) ? "" : logMsg); continue; } // Otherwise, we actually did something. else { readsOut += fend - fbgn; outClr->setbgn(id) = fbgn; outClr->setend(id) = fend; assert(ibgn <= fbgn); assert(fend <= iend); if (fbgn - ibgn > 0) trim5 += fbgn - ibgn; if (iend - fend > 0) trim3 += iend - fend; fprintf(logFile, F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\tMOD%s\n", id, ibgn, iend, fbgn, fend, (logMsg[0] == 0) ? "" : logMsg); } } // Clean up. gkp->gkStore_close(); delete ovs; delete iniClr; delete maxClr; delete outClr; if (logFile) fclose(logFile); // should fprintf() the numbers directly here so an explanation of each category can be supplied; // simpler for now to have report() do it. // Dump the statistics and plots if (outputPrefix) { sprintf(sumName, "%s.stats", outputPrefix); errno = 0; staFile = fopen(sumName, "w"); if (errno) fprintf(stderr, "Failed to open stats file '%s' for writing: %s\n", sumName, strerror(errno)), exit(1); } if (staFile == NULL) staFile = stderr; fprintf(staFile, "PARAMETERS:\n"); fprintf(staFile, "----------\n"); fprintf(staFile, "%7u (reads trimmed below this many bases are deleted)\n", minReadLength); fprintf(staFile, "%7.4f (use overlaps at or below this fraction error)\n", AS_OVS_decodeEvalue(errorValue)); fprintf(staFile, "%7u (break region if overlap is less than this long, for 'largest covered' algorithm)\n", minEvidenceOverlap); fprintf(staFile, "%7u (break region if overlap coverage is less than this many read%s, for 'largest covered' algorithm)\n", minEvidenceCoverage, (minEvidenceCoverage == 1) ? "" : "s"); fprintf(staFile, "\n"); fprintf(staFile, "INPUT READS:\n"); fprintf(staFile, "-----------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (reads processed)\n", readsIn.nReads, readsIn.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (reads not processed, previously deleted)\n", deletedIn.nReads, deletedIn.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (reads not processed, in a library where trimming isn't allowed)\n", noTrimIn.nReads, noTrimIn.nBases); readsIn .generatePlots(outputPrefix, "inputReads", 250); deletedIn.generatePlots(outputPrefix, "inputDeletedReads", 250); noTrimIn .generatePlots(outputPrefix, "inputNoTrimReads", 250); fprintf(staFile, "\n"); fprintf(staFile, "OUTPUT READS:\n"); fprintf(staFile, "------------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (trimmed reads output)\n", readsOut.nReads, readsOut.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (reads with no change, kept as is)\n", noChangeOut.nReads, noChangeOut.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (reads with no overlaps, deleted)\n", noOvlOut.nReads, noOvlOut.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (reads with short trimmed length, deleted)\n", deletedOut.nReads, deletedOut.nBases); readsOut .generatePlots(outputPrefix, "outputTrimmedReads", 250); noOvlOut .generatePlots(outputPrefix, "outputNoOvlReads", 250); deletedOut .generatePlots(outputPrefix, "outputDeletedReads", 250); noChangeOut.generatePlots(outputPrefix, "outputUnchangedReads", 250); fprintf(staFile, "\n"); fprintf(staFile, "TRIMMING DETAILS:\n"); fprintf(staFile, "----------------\n"); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (bases trimmed from the 5' end of a read)\n", trim5.nReads, trim5.nBases); fprintf(staFile, "%6"F_U32P" reads %12"F_U64P" bases (bases trimmed from the 3' end of a read)\n", trim3.nReads, trim3.nBases); trim5.generatePlots(outputPrefix, "trim5", 25); trim3.generatePlots(outputPrefix, "trim3", 25); if ((staFile) && (staFile != stderr)) fclose(staFile); // Buh-bye. exit(0); } canu-1.3/src/overlapBasedTrimming/trimReads.H000066400000000000000000000047061272335422300212650ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2015-JUN-16 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef TRIM_READS_H #define TRIM_READS_H #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include "intervalList.H" #define OBT_MODE_WIGGLE (5) #define OBT_CQ_LENGTH (100) #define OBT_CQO_LENGTH (200) #define OBT_CQO_OVERLAP (100) #define OBT_CQ_SHORT (5) #define OBT_QLT_CLOSE_5 (10) // 5,6 use 5'mode, use 5'min>1 #define OBT_QLT_FAR_5 (50) // 11 use min5' #define OBT_QLT_MODE3 (150) // 9 use 3'mode #define OBT_QLT_CLOSE_MAXM3 (30) // 14 use max>1 close to max #define OBT_QLT_CLOSE_MAX3 (100) // 12 use max3' bool largestCovered(ovOverlap *ovl, uint32 ovlLen, gkRead *read, uint32 ibgn, uint32 iend, uint32 &fbgn, uint32 &fend, char *logMsg, uint32 errorRate, uint32 minOverlap, uint32 minCoverage, uint32 minReadLength); bool bestEdge(ovOverlap *ovl, uint32 ovlLen, gkRead *read, uint32 ibgn, uint32 iend, uint32 &fbgn, uint32 &fend, char *logMsg, uint32 errorRate, uint32 minOverlap, uint32 minCoverage, uint32 minReadLength); #endif // TRIM_READS_H canu-1.3/src/overlapBasedTrimming/trimReads.mk000066400000000000000000000011131272335422300214720ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := trimReads SOURCES := trimReads.C \ trimReads-bestEdge.C \ trimReads-largestCovered.C \ trimReads-quality.C SRC_INCDIRS := .. ../AS_UTL ../stores TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/overlapBasedTrimming/trimStat.H000066400000000000000000000053361272335422300211420ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-JAN-19 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef TRIM_STAT_H #define TRIM_STAT_H #include "AS_global.H" class trimStat { public: trimStat() { nReads = 0; nBases = 0; }; trimStat &operator+=(uint32 bases) { nReads += 1; nBases += bases; histo.push_back(bases); return(*this); }; void generatePlots(char *outputPrefix, char *outputName, uint32 binwidth) { char N[FILENAME_MAX]; FILE *F; sprintf(N, "%s.%s.dat", outputPrefix, outputName); F = fopen(N, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", N, strerror(errno)), exit(1); for (uint64 ii=0; ii /dev/null 2>&1", outputPrefix, outputName); system(N); }; uint32 nReads; uint64 nBases; vector histo; }; #endif // TRIM_STAT_H canu-1.3/src/overlapErrorAdjustment/000077500000000000000000000000001272335422300176165ustar00rootroot00000000000000canu-1.3/src/overlapErrorAdjustment/analyzeAlignment.C000066400000000000000000000472261272335422300232370ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-JUN-18 to 2015-JUL-01 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "analyzeAlignment.H" // Return the substitution vote corresponding to Ch . static Vote_Value_t Matching_Vote(char ch) { switch (ch) { case 'A': return(A_SUBST); break; case 'C': return(C_SUBST); break; case 'G': return(G_SUBST); break; case 'T': return(T_SUBST); break; } fprintf(stderr, "Matching_Vote()-- invalid letter '%c'\n", ch); return(NO_VOTE); } static char Matching_Char(Vote_Value_t vv) { switch (vv) { case A_SUBST: return('A'); break; case C_SUBST: return('C'); break; case G_SUBST: return('G'); break; case T_SUBST: return('T'); break; default: return('?'); break; } return('?'); } // This is expecting: // aSeq and bSeq to be pointers to the start of the sequence that was aligned with Prefix_Edit_Distance // aLen and bLen to be the strlen of those strings // aOffset to be the offset from base zero of the read // deltaLen and delta as passed back from the aligner // void analyzeAlignment::analyze(char *aSeq, int32 aLen, int32 aOffset, char *bSeq, int32 bLen, int32 deltaLen, int32 *delta) { assert(aLen >= 0); assert(bLen >= 0); int32 ct = 0; _readSub[ct] = -1; _algnSub[ct] = -1; _voteValue[ct] = A_SUBST; // Dummy value ct++; int32 i = 0; int32 j = 0; int32 p = 0; uint32 nMatch = 0; uint32 nMismatch = 0; uint32 nInsert = 0; uint32 nDelete = 0; for (int32 k=0; k 0) { _readSub[ct] = i; _algnSub[ct] = p; _voteValue[ct] = DELETE; #ifdef DEBUG fprintf(stderr, "DELETE %c at %d #%d\n", aSeq[i], i, p); #endif nDelete++; ct++; i++; assert(i <= aLen); p++; } #ifdef DEBUG fprintf(stderr, "#4 match %u mismatch %u insert %u delete %u\n", nMatch, nMismatch, nInsert, nDelete); #endif } // No more deltas. While there is still sequence, add matches or mismatches. #ifdef DEBUG fprintf(stderr, "#5 match %u mismatch %u insert %u delete %u\n", nMatch, nMismatch, nInsert, nDelete); fprintf(stderr, "#5 k=DONE i=%d out of %d j=%d out of %d\n", i, aLen, j, bLen); #endif while (i < aLen) { //fprintf(stderr, "k=DONE i=%d out of %d j=%d out of %d\n", i, aLen, j, bLen); if (aSeq[i] != bSeq[j]) nMismatch++; else nMatch++; if (aSeq[i] != bSeq[j]) { _readSub[ct] = i; _algnSub[ct] = p; switch (bSeq[j]) { case 'A': _voteValue[ct] = A_SUBST; break; case 'C': _voteValue[ct] = C_SUBST; break; case 'G': _voteValue[ct] = G_SUBST; break; case 'T': _voteValue[ct] = T_SUBST; break; default : fprintf(stderr, "ERROR:[3] Bad sequence '%c' 0x%02x)\n", bSeq[j], bSeq[j]); assert(0); } ct++; } i++; assert(i <= aLen); // Guaranteed, we're looping on this j++; assert(j <= bLen); p++; } #ifdef DEBUG fprintf(stderr, "#6 match %u mismatch %u insert %u delete %u\n", nMatch, nMismatch, nInsert, nDelete); #endif _readSub[ct] = i; _algnSub[ct] = p; // For each identified change, add votes for some region around the change. //fprintf(stderr, "Found %u changes.\n", ct); for (int32 i=1; i<=ct; i++) { int32 prev_match = _algnSub[i] - _algnSub[i-1] - 1; int32 p_lo = (i == 1 ? 0 : End_Exclude_Len); int32 p_hi = (i == ct ? prev_match : prev_match - End_Exclude_Len); // If distance to previous match is bigger than 'kmer' size, make a new vote. // This operates one ahead of where votes are added - we add votes for _readSub[i-1] when at [i]. if (prev_match >= Kmer_Len) { fprintf(stderr, "adjust ct %d pos %d - lo %d hi %d\n", i, _readSub[i-1], p_lo, p_hi); fprintf(stderr, " match vote %u to %u\n", aOffset + _readSub[i-1] + 1, aOffset + _readSub[i-1] + p_lo + 1); for (int32 p=0; p 0) || (_voteValue[i-1] <= T_SUBST) || (_voteValue[i] <= T_SUBST))) { int32 next_match = _algnSub[i+1] - _algnSub[i] - 1; if (prev_match + next_match >= Vote_Qualify_Len) castVote(_voteValue[i], aOffset + _readSub[i]); } } } void analyzeAlignment::outputDetails(uint32 j) { fprintf(stderr, "%3"F_U32P": %c conf %3"F_U64P" deletes %3"F_U64P" | subst %3"F_U64P" %3"F_U64P" %3"F_U64P" %3"F_U64P" | no_insert %3"F_U64P" insert %3"F_U64P" %3"F_U64P" %3"F_U64P" %3"F_U64P"\n", j, _seq[j], _vote[j].confirmed, _vote[j].deletes, _vote[j].a_subst, _vote[j].c_subst, _vote[j].g_subst, _vote[j].t_subst, _vote[j].no_insert, _vote[j].a_insert, _vote[j].c_insert, _vote[j].g_insert, _vote[j].t_insert); }; void analyzeAlignment::outputDetails(void) { fprintf(stderr, ">%d\n", _readID); for (uint32 j=0; _seq[j] != '\0'; j++) outputDetails(j); }; void analyzeAlignment::generateCorrections(FILE *corFile) { _corLen = 0; _cor[_corLen].keep_left = (_lDegree < Degree_Threshold); _cor[_corLen].keep_right = (_rDegree < Degree_Threshold); _cor[_corLen].type = IDENT; _cor[_corLen].pos = 0; _cor[_corLen].readID = _readID; _corLen++; resizeArray(_cor, _corLen, _corMax, _corLen+1); uint32 passedLowConfirmed = 0; uint32 substitutions = 0; uint32 skippedTooFew = 0; // 0 or 1 votes uint32 skippedTooWeak = 0; // No vote more than 50% uint32 skippedNoChange = 0; // Is a substitution vote, but it's the same as the base that is there uint32 skippedHaplo = 0; // More than one significant vote, and we're not correcting haplotypes uint32 skippedConfirmed = 0; // ?? uint32 passedInsert = 0; uint32 insertions = 0; uint32 skippedInsTotal = 0; uint32 skippedInsMax = 0; uint32 skippedInsHaplo = 0; uint32 skippedInsTooMany = 0; for (uint32 j=0; j<_seqLen; j++) { outputDetails(j); if (_vote[j].confirmed < 2) { Vote_Value_t vval = DELETE; uint64 max = _vote[j].deletes; bool is_change = true; passedLowConfirmed++; if (_vote[j].a_subst > max) { vval = A_SUBST; max = _vote[j].a_subst; is_change = (_seq[j] != 'A'); } if (_vote[j].c_subst > max) { vval = C_SUBST; max = _vote[j].c_subst; is_change = (_seq[j] != 'C'); } if (_vote[j].g_subst > max) { vval = G_SUBST; max = _vote[j].g_subst; is_change = (_seq[j] != 'G'); } if (_vote[j].t_subst > max) { vval = T_SUBST; max = _vote[j].t_subst; is_change = (_seq[j] != 'T'); } uint64 haplo_ct = ((_vote[j].deletes >= Min_Haplo_Occurs) + (_vote[j].a_subst >= Min_Haplo_Occurs) + (_vote[j].c_subst >= Min_Haplo_Occurs) + (_vote[j].g_subst >= Min_Haplo_Occurs) + (_vote[j].t_subst >= Min_Haplo_Occurs)); uint64 total = (_vote[j].deletes + _vote[j].a_subst + _vote[j].c_subst + _vote[j].g_subst + _vote[j].t_subst); // The original had a gargantuan if test (five clauses, all had to be true) to decide if a // record should be output. It was negated into many small tests if we should skip the // output. A side effect is that we can abort a little earlier (skipping the two clauses // above)....but we don't bother. // (total > 1) if (total <= 1) { fprintf(stderr, "FEW total = "F_U64" <= 1\n", total); skippedTooFew++; continue; } // (2 * max > total) if (2 * max <= total) { fprintf(stderr, "WEAK 2*max = "F_U64" <= total = "F_U64"\n", 2*max, total); skippedTooWeak++; continue; } // (is_change == true) if (is_change == false) { fprintf(stderr, "SAME is_change = %s\n", (is_change) ? "true" : "false"); skippedNoChange++; continue; } // ((haplo_ct < 2) || (Use_Haplo_Ct == false)) if ((haplo_ct >= 2) && (Use_Haplo_Ct == true)) { fprintf(stderr, "HAPLO haplo_ct="F_U64" >= 2 AND Use_Haplo_Ct = %s\n", haplo_ct, (Use_Haplo_Ct) ? "true" : "false"); skippedHaplo++; continue; } // ((_vote[j].confirmed == 0) || // ((_vote[j].confirmed == 1) && (max > 6))) if ((_vote[j].confirmed > 0) && ((_vote[j].confirmed != 1) || (max <= 6))) { fprintf(stderr, "INDET confirmed = "F_U64" max = "F_U64"\n", _vote[j].confirmed, max); skippedConfirmed++; continue; } // Otherwise, output. substitutions++; fprintf(stderr, "SUBSTITUTE position "F_U32" to %c\n", j, Matching_Char(vval)); _cor[_corLen].type = vval; _cor[_corLen].pos = j; _cor[_corLen].readID = _readID; _corLen++; resizeArray(_cor, _corLen, _corMax, _corLen+1); } // confirmed < 2 if (_vote[j].no_insert < 2) { Vote_Value_t ins_vote = A_INSERT; uint64 ins_max = _vote[j].a_insert; passedInsert++; if (ins_max < _vote[j].c_insert) { ins_vote = C_INSERT; ins_max = _vote[j].c_insert; } if (ins_max < _vote[j].g_insert) { ins_vote = G_INSERT; ins_max = _vote[j].g_insert; } if (ins_max < _vote[j].t_insert) { ins_vote = T_INSERT; ins_max = _vote[j].t_insert; } uint64 ins_haplo_ct = ((_vote[j].a_insert >= Min_Haplo_Occurs) + (_vote[j].c_insert >= Min_Haplo_Occurs) + (_vote[j].g_insert >= Min_Haplo_Occurs) + (_vote[j].t_insert >= Min_Haplo_Occurs)); uint64 ins_total = (_vote[j].a_insert + _vote[j].c_insert + _vote[j].g_insert + _vote[j].t_insert); if (ins_total <= 1) { fprintf(stderr, "FEW ins_total = "F_U64" <= 1\n", ins_total); skippedInsTotal++; continue; } if (2 * ins_max >= ins_total) { fprintf(stderr, "WEAK 2*ins_max = "F_U64" <= ins_total = "F_U64"\n", 2*ins_max, ins_total); skippedInsMax++; continue; } if ((ins_haplo_ct >= 2) && (Use_Haplo_Ct == true)) { fprintf(stderr, "HAPLO ins_haplo_ct="F_U64" >= 2 AND Use_Haplo_Ct = %s\n", ins_haplo_ct, (Use_Haplo_Ct) ? "true" : "false"); skippedInsHaplo++; continue; } if ((_vote[j].no_insert > 0) && ((_vote[j].no_insert != 1) || (ins_max <= 6))) { fprintf(stderr, "INDET no_insert = "F_U64" ins_max = "F_U64"\n", _vote[j].no_insert, ins_max); skippedInsTooMany++; continue; } // Otherwise, output. insertions++; fprintf(stderr, "INSERT position "F_U32" to %c\n", j, Matching_Char(ins_vote)); _cor[_corLen].type = ins_vote; _cor[_corLen].pos = j; _cor[_corLen].readID = _readID; _corLen++; resizeArray(_cor, _corLen, _corMax, _corLen+1); } // insert < 2 } fprintf(stderr, "Processed corrections: made %6u subs and %6u inserts - possible %6u (few %6u weak %6u same %6u haplo %6u confirmed %6u) inserts %6u (total %6u max %6u haplo %6u confirmed %6u)\n", substitutions, insertions, passedLowConfirmed, skippedTooFew, skippedTooWeak, skippedNoChange, skippedHaplo, skippedConfirmed, passedInsert, skippedInsTotal, skippedInsMax, skippedInsHaplo, skippedInsTooMany); if (corFile) AS_UTL_safeWrite(corFile, _cor, "corrections", sizeof(Correction_Output_t), _corLen); } #if 0 fprintf(stderr, "Corrected "F_U64" bases with "F_U64" substitutions, "F_U64" deletions and "F_U64" insertions.\n", G->basesLen, changes[A_SUBST] + changes[C_SUBST] + changes[G_SUBST] + changes[T_SUBST], changes[DELETE], changes[A_INSERT] + changes[C_INSERT] + changes[G_INSERT] + changes[T_INSERT]); #endif void analyzeAlignment::generateCorrectedRead(Adjust_t *fadj, uint32 *fadjLen, uint64 *changes) { // oseq = original sequence // fseq - fixed sequence _corSeqLen = 0; uint32 corPos = 1; // First entry is the ident block. uint32 adjVal = 0; // Corrected reads start at position zero, really! for (uint32 i=0; i<_seqLen; i++) { // No more corrections, or no more corrections for this read -- just copy bases till the end. if ((corPos == _corLen) || (_cor[corPos].readID != _readID)) { //fprintf(stderr, "no more corrections at i=%u, copy rest of read as is\n", i); while (i < _seqLen) _corSeq[_corSeqLen++] = _filter[_seq[i++]]; break; } // Not at a correction -- copy the base. if (i < _cor[corPos].pos) { _corSeq[_corSeqLen++] = _filter[_seq[i]]; continue; } if ((i != _cor[corPos].pos) && (i != _cor[corPos].pos + 1)) fprintf(stderr, "i=%d corPos=%d _cor[corPos].pos=%d\n", i, corPos, _cor[corPos].pos); assert((i == _cor[corPos].pos) || (i == _cor[corPos].pos + 1)); if (changes) changes[_cor[corPos].type]++; switch (_cor[corPos].type) { case DELETE: // Delete base //fprintf(stderr, "DELETE %u pos %u adjust %d\n", (*fadjLen), i+1, adjVal-1); if (fadj) { fadj[(*fadjLen)].adjpos = i + 1; fadj[(*fadjLen)].adjust = --adjVal; (*fadjLen)++; } break; case A_SUBST: _corSeq[_corSeqLen++] = 'A'; break; case C_SUBST: _corSeq[_corSeqLen++] = 'C'; break; case G_SUBST: _corSeq[_corSeqLen++] = 'G'; break; case T_SUBST: _corSeq[_corSeqLen++] = 'T'; break; case A_INSERT: if (i != _cor[corPos].pos + 1) { // Insert not immediately after subst //fprintf(stderr, "A i=%d != _cor[%d].pos+1=%d\n", i, corPos, _cor[corPos].pos+1); _corSeq[_corSeqLen++] = _filter[_seq[i++]]; } _corSeq[_corSeqLen++] = 'A'; if (fadj) { fadj[(*fadjLen)].adjpos = i + 1; fadj[(*fadjLen)].adjust = ++adjVal; (*fadjLen)++; } i--; // Undo the automagic loop increment break; case C_INSERT: if (i != _cor[corPos].pos + 1) { //fprintf(stderr, "C i=%d != _cor[%d].pos+1=%d\n", i, corPos, _cor[corPos].pos+1); _corSeq[_corSeqLen++] = _filter[_seq[i++]]; } _corSeq[_corSeqLen++] = 'C'; if (fadj) { fadj[(*fadjLen)].adjpos = i + 1; fadj[(*fadjLen)].adjust = ++adjVal; (*fadjLen)++; } i--; break; case G_INSERT: if (i != _cor[corPos].pos + 1) { //fprintf(stderr, "G i=%d != _cor[%d].pos+1=%d\n", i, corPos, _cor[corPos].pos+1); _corSeq[_corSeqLen++] = _filter[_seq[i++]]; } _corSeq[_corSeqLen++] = 'G'; if (fadj) { fadj[(*fadjLen)].adjpos = i + 1; fadj[(*fadjLen)].adjust = ++adjVal; (*fadjLen)++; } i--; break; case T_INSERT: if (i != _cor[corPos].pos + 1) { //fprintf(stderr, "T i=%d != _cor[%d].pos+1=%d\n", i, corPos, _cor[corPos].pos+1); _corSeq[_corSeqLen++] = _filter[_seq[i++]]; } _corSeq[_corSeqLen++] = 'T'; if (fadj) { fadj[(*fadjLen)].adjpos = i + 1; fadj[(*fadjLen)].adjust = ++adjVal; (*fadjLen)++; } i--; break; default: fprintf (stderr, "ERROR: Illegal vote type\n"); break; } corPos++; } // Terminate the sequence. _corSeq[_corSeqLen] = 0; } canu-1.3/src/overlapErrorAdjustment/analyzeAlignment.H000066400000000000000000000147361272335422300232440ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-JUN-18 to 2015-JUL-01 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "correctionOutput.H" // Allocate one per base in read under evaluation. // 5 values per 64 bit word -> 12 bits per value + 4 left over // 6 values per 64 bit word -> 10 bits per value + 4 left over // 7 values per 64 bit word -> 9 bits per value + 1 left over // // Since there are only 11 values to store, we use the 10 bit size. // // This is only in-memory, so we don't worry about padding it to maintain // proper sizes between runs. #define VOTETALLY_BITS 10 #define VOTETALLY_MAX 1023 struct Vote_Tally_t { uint64 confirmed : VOTETALLY_BITS; uint64 deletes : VOTETALLY_BITS; uint64 a_subst : VOTETALLY_BITS; uint64 c_subst : VOTETALLY_BITS; uint64 g_subst : VOTETALLY_BITS; uint64 t_subst : VOTETALLY_BITS; uint64 no_insert : VOTETALLY_BITS; uint64 a_insert : VOTETALLY_BITS; uint64 c_insert : VOTETALLY_BITS; uint64 g_insert : VOTETALLY_BITS; uint64 t_insert : VOTETALLY_BITS; }; // Maps an original location to a fixed location. Supposedly // smaller/faster than the obvious array mapping... struct Adjust_t { int32 adjpos; int32 adjust; }; class analyzeAlignment { public: analyzeAlignment() { Degree_Threshold = 2; // ?? Use_Haplo_Ct = true; // Use haplotype counts to correct End_Exclude_Len = 3; // ?? Kmer_Len = 9; // ?? Vote_Qualify_Len = 9; // ?? Min_Haplo_Occurs = 3; // This many or more votes for the same base indicates a haplotype _readID = 0; _seqLen = 0; _seq = NULL; _corSeqLen = 0; _corSeqMax = AS_MAX_READLEN; _corSeq = new char [_corSeqMax]; _voteMax = 0; _vote = NULL; _lDegree = 0; _rDegree = 0; _readSub = new int32 [AS_MAX_READLEN]; _algnSub = new int32 [AS_MAX_READLEN]; _voteValue = new Vote_Value_t [AS_MAX_READLEN]; _corLen = 0; _corMax = AS_MAX_READLEN / 4; _cor = new Correction_Output_t [_corMax]; // The original version converted ACGT to lowercase, and replaced non-ACGT with 'a'. for (uint32 i=0; i<256; i++) _filter[i] = 'a'; _filter['A'] = _filter['a'] = 'a'; _filter['C'] = _filter['c'] = 'c'; _filter['G'] = _filter['g'] = 'g'; _filter['T'] = _filter['t'] = 't'; }; ~analyzeAlignment() { delete [] _corSeq; delete [] _vote; delete [] _readSub; delete [] _algnSub; delete [] _voteValue; delete [] _cor; }; public: void reset(uint32 id, char *seq, uint32 seqLen) { fprintf(stderr, "reset() for read id %u of length %u\n", id, seqLen); _readID = id; _seqLen = seqLen; _seq = seq; _corSeqLen = 0; if (_voteMax < _seqLen) { delete [] _vote; _voteMax = _seqLen + 2000; _vote = new Vote_Tally_t [_voteMax]; } // Seems like overkill, but I don't know where to put it memset(_vote, 0, sizeof(Vote_Tally_t) * _voteMax); }; void analyze(char *a, int32 aLen, int32 aOffset, char *b, int32 bLen, int32 deltaLen, int32 *delta); void outputDetails(uint32 j); void outputDetails(void); void generateCorrections(FILE *corFile=NULL); void generateCorrectedRead(Adjust_t *fadj=NULL, uint32 *fadjLen=NULL, uint64 *changes=NULL); private: void castVote(Vote_Value_t val, int32 pos) { int32 v = 0; switch (val) { case DELETE: if (_vote[pos].deletes < VOTETALLY_MAX) v = ++_vote[pos].deletes; break; case A_SUBST: if (_vote[pos].a_subst < VOTETALLY_MAX) v = ++_vote[pos].a_subst; break; case C_SUBST: if (_vote[pos].c_subst < VOTETALLY_MAX) v = ++_vote[pos].c_subst; break; case G_SUBST: if (_vote[pos].g_subst < VOTETALLY_MAX) v = ++_vote[pos].g_subst; break; case T_SUBST: if (_vote[pos].t_subst < VOTETALLY_MAX) v = ++_vote[pos].t_subst; break; case A_INSERT: if (_vote[pos].a_insert < VOTETALLY_MAX) v = ++_vote[pos].a_insert; break; case C_INSERT: if (_vote[pos].c_insert < VOTETALLY_MAX) v = ++_vote[pos].c_insert; break; case G_INSERT: if (_vote[pos].g_insert < VOTETALLY_MAX) v = ++_vote[pos].g_insert; break; case T_INSERT: if (_vote[pos].t_insert < VOTETALLY_MAX) v = ++_vote[pos].t_insert; break; case NO_VOTE: break; default : fprintf(stderr, "ERROR: Illegal vote type\n"); break; } }; private: // Parameters int32 Degree_Threshold; int32 Use_Haplo_Ct; int32 End_Exclude_Len; int32 Kmer_Len; int32 Vote_Qualify_Len; int32 Min_Haplo_Occurs; // Per-read data uint32 _readID; uint32 _seqLen; char *_seq; public: uint32 _corSeqLen; uint32 _corSeqMax; char *_corSeq; private: uint32 _voteMax; Vote_Tally_t *_vote; uint32 _lDegree; uint32 _rDegree; int32 *_readSub; int32 *_algnSub; Vote_Value_t *_voteValue; // Outputs (used to generate the corrected read) uint32 _corLen; uint32 _corMax; Correction_Output_t *_cor; char _filter[256]; }; canu-1.3/src/overlapErrorAdjustment/correctOverlaps-Correct_Frags.C000066400000000000000000000230741272335422300256260ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAY-20 to 2015-JUN-03 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-MAY-02 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "correctOverlaps.H" #include "correctionOutput.H" // Shouldn't be global. char filter[256]; void correctRead(uint32 curID, char *fseq, uint32 &fseqLen, Adjust_t *fadj, uint32 &fadjLen, char *oseq, uint32 oseqLen, Correction_Output_t *C, uint64 &Cpos, uint64 Clen, uint64 *changes) { #if DO_NO_CORRECTIONS // for testing if the adjustments are screwed up. yup. strcpy(fseq, oseq); fseqLen += oseqLen; return; #endif //fprintf(stderr, "Correcting read %u\n", curID); // Find the correct corrections. while ((Cpos < Clen) && (C[Cpos].readID < curID)) { //fprintf(stderr, "SKIP Cpos=%d for read %u, want read %u\n", Cpos, C[Cpos].readID, curID); Cpos++; } // Skip any IDENT message. assert(C[Cpos].type == IDENT); //G.reads[G.readsLen].keep_left = C[Cpos].keep_left; //G.reads[G.readsLen].keep_right = C[Cpos].keep_right; Cpos++; //fprintf(stderr, "Start at Cpos=%d position=%d type=%d id=%d\n", Cpos, C[Cpos].pos, C[Cpos].type, C[Cpos].readID); int32 adjVal = 0; for (uint32 i=0; i%u\n%s\n", curID, fseq); } // Open and read corrections from Correct_File_Path and // apply them to sequences in Frag . // Load reads from gkpStore, and apply corrections. void Correct_Frags(coParameters *G, gkStore *gkpStore) { // The original converted to lowercase, and made non-acgt be 'a'. for (uint32 i=0; i<256; i++) filter[i] = 'a'; filter['A'] = filter['a'] = 'a'; filter['C'] = filter['c'] = 'c'; filter['G'] = filter['g'] = 'g'; filter['T'] = filter['t'] = 't'; // Open the corrections, as an array. memoryMappedFile *Cfile = new memoryMappedFile(G->correctionsName); Correction_Output_t *C = (Correction_Output_t *)Cfile->get(); uint64 Cpos = 0; uint64 Clen = Cfile->length() / sizeof(Correction_Output_t); uint64 firstRecord = 0; uint64 currentRecord = 0; fprintf(stderr, "Reading "F_U64" corrections from '%s'.\n", Clen, G->correctionsName); // Count the number of bases, so we can do two gigantic allocations for bases and adjustments. // Adjustments are always less than the number of corrections; we could also count exactly. G->basesLen = 0; G->adjustsLen = 0; for (uint32 curID=G->bgnID; curID<=G->endID; curID++) { gkRead *read = gkpStore->gkStore_getRead(curID); G->basesLen += read->gkRead_sequenceLength() + 1; } for (uint64 c=0; cadjustsLen++; break; } } fprintf(stderr, "Correcting "F_U64" bases with "F_U64" indel adjustments.\n", G->basesLen, G->adjustsLen); fprintf(stderr, "--Allocate "F_U64" + "F_U64" + "F_U64" MB for bases, adjusts and reads.\n", (sizeof(char) * G->basesLen) >> 20, (sizeof(Adjust_t) * G->adjustsLen) >> 20, (sizeof(Frag_Info_t) * (G->endID - G->bgnID + 1)) >> 20); G->bases = new char [G->basesLen]; G->adjusts = new Adjust_t [G->adjustsLen]; G->reads = new Frag_Info_t [G->endID - G->bgnID + 1]; G->readsLen = 0; G->basesLen = 0; G->adjustsLen = 0; uint64 changes[12] = {0}; // Load reads and apply corrections for each one. gkReadData *readData = new gkReadData; for (uint32 curID=G->bgnID; curID<=G->endID; curID++) { gkRead *read = gkpStore->gkStore_getRead(curID); gkpStore->gkStore_loadReadData(read, readData); uint32 readLength = read->gkRead_sequenceLength(); char *readBases = readData->gkReadData_getSequence(); // Save pointers to the bases and adjustments. G->reads[G->readsLen].bases = G->bases + G->basesLen; G->reads[G->readsLen].basesLen = 0; G->reads[G->readsLen].adjusts = G->adjusts + G->adjustsLen; G->reads[G->readsLen].adjustsLen = 0; // Find the correct corrections. while ((Cpos < Clen) && (C[Cpos].readID < curID)) Cpos++; // We should be at the IDENT message. if (C[Cpos].type != IDENT) { fprintf(stderr, "ERROR: didn't find IDENT at Cpos="F_U64" for read "F_U32"\n", Cpos, curID); fprintf(stderr, " C[Cpos] = keep_left=%u keep_right=%u type=%u pos=%u readID=%u\n", C[Cpos].keep_left, C[Cpos].keep_right, C[Cpos].type, C[Cpos].pos, C[Cpos].readID); } assert(C[Cpos].type == IDENT); G->reads[G->readsLen].keep_left = C[Cpos].keep_left; G->reads[G->readsLen].keep_right = C[Cpos].keep_right; //Cpos++; // Now do the corrections. correctRead(curID, G->reads[G->readsLen].bases, G->reads[G->readsLen].basesLen, G->reads[G->readsLen].adjusts, G->reads[G->readsLen].adjustsLen, readData->gkReadData_getSequence(), read->gkRead_sequenceLength(), C, Cpos, Clen, changes); // Update the lengths in the globals. G->basesLen += G->reads[G->readsLen].basesLen + 1; G->adjustsLen += G->reads[G->readsLen].adjustsLen; G->readsLen += 1; } delete readData; delete Cfile; fprintf(stderr, "Corrected "F_U64" bases with "F_U64" substitutions, "F_U64" deletions and "F_U64" insertions.\n", G->basesLen, changes[A_SUBST] + changes[C_SUBST] + changes[G_SUBST] + changes[T_SUBST], changes[DELETE], changes[A_INSERT] + changes[C_INSERT] + changes[G_INSERT] + changes[T_INSERT]); } canu-1.3/src/overlapErrorAdjustment/correctOverlaps-Prefix_Edit_Distance.C000066400000000000000000000203631272335422300271150ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAY-02 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "correctOverlaps.H" static void Compute_Delta(pedWorkArea_t *WA, int32 e, int32 d, int32 row) { int32 last = row; int32 stackLen = 0; for (int32 k=e; k>0; k--) { int32 from = d; int32 max = 1 + WA->Edit_Array_Lazy[k-1][d]; int32 j = WA->Edit_Array_Lazy[k-1][d-1]; if (j > max) { from = d-1; max = j; } j = 1 + WA->Edit_Array_Lazy[k-1][d+1]; if (j > max) { from = d+1; max = j; } if (from == d-1) { WA->deltaStack[stackLen++] = max - last - 1; d--; last = WA->Edit_Array_Lazy[k-1][from]; } else if (from == d+1) { WA->deltaStack[stackLen++] = last - (max - 1); d++; last = WA->Edit_Array_Lazy[k-1][from]; } } WA->deltaStack[stackLen++] = last + 1; for (int32 k=0, i=stackLen-1; i>0; i--) WA->delta[k++] = abs(WA->deltaStack[i]) * Sign(WA->deltaStack[i-1]); WA->deltaLen = stackLen - 1; } // Allocate another block of 64mb for edits // Needs to be at least: // 52,432 to handle 40% error at 64k overlap // 104,860 to handle 80% error at 64k overlap // 209,718 to handle 40% error at 256k overlap // 419,434 to handle 80% error at 256k overlap // 3,355,446 to handle 40% error at 4m overlap // 6,710,890 to handle 80% error at 4m overlap // Bigger means we can assign more than one Edit_Array[] in one allocation. uint32 EDIT_SPACE_SIZE = 16 * 1024 * 1024; static void Allocate_More_Edit_Space(pedWorkArea_t *WA) { // Determine the last allocated block, and the last assigned block int32 b = 0; // Last edit array assigned int32 e = 0; // Last edit array assigned more space int32 a = WA->alloc.size(); // First unallocated block while (WA->Edit_Array_Lazy[b] != NULL) b++; // Fill in the edit space array. Well, not quite yet. First, decide the minimum size. // // Element [0] can access from [-2] to [2] = 5 elements. // Element [1] can access from [-3] to [3] = 7 elements. // // Element [e] can access from [-2-e] to [2+e] = 5 + e * 2 elements // // So, our offset for this new block needs to put [e][0] at offset... int32 Offset = 2 + b; int32 Del = 6 + b * 2; int32 Size = EDIT_SPACE_SIZE; while (Size < Offset + Del) Size *= 2; // Allocate another block int32 *alloc = new int32 [Size]; WA->alloc.push_back(alloc); // And, now, fill in the edit space array. e = b; while ((Offset + Del < Size) && (e < WA->Edit_Array_Max)) { WA->Edit_Array_Lazy[e++] = alloc + Offset; Offset += Del; Del += 2; } if (e == b) fprintf(stderr, "Allocate_More_Edit_Space()-- ERROR: couldn't allocate enough space for even one more entry! e=%d\n", e); assert(e != b); fprintf(stderr, "--Allocate "F_U64" MB for edit array work space %u (positions %u-%u)\n", Size >> 20, a, b, e-1); } // Return the minimum number of changes (inserts, deletes, replacements) // needed to match string A[0 .. (m-1)] with a prefix of string // T[0 .. (n-1)] if it's not more than Error_Limit . // // Put delta description of alignment in WA->delta and set // WA->deltaLen to the number of entries there if it's a complete // match. // Set A_End and T_End to the rightmost positions where the // alignment ended in A and T , respectively. // Set Match_To_End true if the match extended to the end // of at least one string; otherwise, set it false to indicate // a branch point. int32 Prefix_Edit_Dist(char *A, int32 m, char *T, int32 n, int32 Error_Limit, int32 &A_End, int32 &T_End, bool &Match_To_End, pedWorkArea_t *WA) { //assert (m <= n); int32 Best_d = 0; int32 Best_e = 0; int32 Longest = 0; WA->deltaLen = 0; int32 shorter = min(m, n); int32 Row = 0; while ((Row < shorter) && (A[Row] == T[Row])) Row++; //fprintf(stderr, "Row=%d matches at the start\n", Row); if (WA->Edit_Array_Lazy[0] == NULL) Allocate_More_Edit_Space(WA); WA->Edit_Array_Lazy[0][0] = Row; // Exact match? if (Row == shorter) { A_End = Row; T_End = Row; Match_To_End = true; return(0); } int32 Left = 0; int32 Right = 0; double Max_Score = 0.0; int32 Max_Score_Len = 0; int32 Max_Score_Best_d = 0; int32 Max_Score_Best_e = 0; for (int32 e=1; e<=Error_Limit; e++) { if (WA->Edit_Array_Lazy[e] == NULL) Allocate_More_Edit_Space(WA); Left = max(Left - 1, -e); Right = min(Right + 1, e); WA->Edit_Array_Lazy[e-1][Left] = -2; WA->Edit_Array_Lazy[e-1][Left-1] = -2; WA->Edit_Array_Lazy[e-1][Right] = -2; WA->Edit_Array_Lazy[e-1][Right+1] = -2; for (int32 d=Left; d<=Right; d++) { Row = 1 + WA->Edit_Array_Lazy[e-1][d]; Row = max(Row, WA->Edit_Array_Lazy[e-1][d-1]); Row = max(Row, WA->Edit_Array_Lazy[e-1][d+1] + 1); while ((Row < m) && (Row + d < n) && (A[Row] == T[Row + d])) Row++; //fprintf(stderr, "Row=%d matches at error e=%d\n", Row, e); assert(e < WA->Edit_Array_Max); WA->Edit_Array_Lazy[e][d] = Row; if (Row == m || Row + d == n) { //fprintf(stderr, "Hit end Row=%d m=%d Row+d=%d n=%d\n", Row, m, Row+d, n); // Force last error to be mismatch rather than insertion if ((Row == m) && (1 + WA->Edit_Array_Lazy[e-1][d+1] == WA->Edit_Array_Lazy[e][d]) && (d < Right)) { d++; WA->Edit_Array_Lazy[e][d] = WA->Edit_Array_Lazy[e][d-1]; } A_End = Row; // One past last align position T_End = Row + d; Match_To_End = true; Compute_Delta(WA, e, d, Row); return(e); } } while (Left <= Right && Left < 0 && WA->Edit_Array_Lazy[e][Left] < WA->G->Edit_Match_Limit[e]) Left++; if (Left >= 0) while (Left <= Right && WA->Edit_Array_Lazy[e][Left] + Left < WA->G->Edit_Match_Limit[e]) Left++; if (Left > Right) break; while (Right > 0 && WA->Edit_Array_Lazy[e][Right] + Right < WA->G->Edit_Match_Limit[e]) Right--; if (Right <= 0) while (WA->Edit_Array_Lazy[e][Right] < WA->G->Edit_Match_Limit[e]) Right--; assert (Left <= Right); for (int32 d=Left; d <= Right; d++) if (WA->Edit_Array_Lazy[e][d] > Longest) { Best_d = d; Best_e = e; Longest = WA->Edit_Array_Lazy[e][d]; } int32 Score = Longest * BRANCH_PT_MATCH_VALUE - e; // Assumes BRANCH_PT_MATCH_VALUE - BRANCH_PT_ERROR_VALUE == 1.0 // findErrors also included a second test; overlapper doesn't. if (Score > Max_Score) { Max_Score = Score; Max_Score_Len = Longest; Max_Score_Best_d = Best_d; Max_Score_Best_e = Best_e; } } // findErrors does this call. Overlapper doesn't. //Compute_Delta(WA, Max_Score_Best_e, Max_Score_Best_d, Max_Score_Len); A_End = Max_Score_Len; T_End = Max_Score_Len + Max_Score_Best_d; Match_To_End = false; // findErrors is returning Max_Score_Best_e. So does overlapper. // The original return was just e, but the only way we get here is if the e loop // exits with e = Error_Limit+1. return(Error_Limit + 1); } canu-1.3/src/overlapErrorAdjustment/correctOverlaps-Read_Olaps.C000066400000000000000000000051121272335422300251050ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-JUN-16 to 2015-JUN-25 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-MAY-02 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "correctOverlaps.H" // Load overlaps with aIID from G->bgnID to G->endID. // Overlaps can be unsorted. void Read_Olaps(coParameters *G, gkStore *gkpStore) { ovStore *ovs = new ovStore(G->ovlStorePath, gkpStore); ovs->setRange(G->bgnID, G->endID); uint64 numolaps = ovs->numOverlapsInRange(); uint64 numNormal = 0; uint64 numInnie = 0; fprintf(stderr, "Read_Olaps()-- Loading "F_U64" overlaps from '%s' for reads "F_U32" to "F_U32"\n", numolaps, G->ovlStorePath, G->bgnID, G->endID); fprintf(stderr, "--Allocate "F_U64" MB for overlaps.\n", (sizeof(Olap_Info_t) * numolaps) >> 20); G->olaps = new Olap_Info_t [numolaps]; G->olapsLen = 0; ovOverlap olap(gkpStore); while (ovs->readOverlap(&olap)) { G->olaps[G->olapsLen].a_iid = olap.a_iid; G->olaps[G->olapsLen].b_iid = olap.b_iid; G->olaps[G->olapsLen].a_hang = olap.a_hang(); G->olaps[G->olapsLen].b_hang = olap.b_hang(); //G->olaps[G->olapsLen].orient = (olap.flipped()) ? INNIE : NORMAL; G->olaps[G->olapsLen].innie = (olap.flipped() == true); G->olaps[G->olapsLen].normal = (olap.flipped() == false); G->olaps[G->olapsLen].order = G->olapsLen; G->olaps[G->olapsLen].evalue = olap.evalue(); numNormal += (G->olaps[G->olapsLen].normal == true); numInnie += (G->olaps[G->olapsLen].innie == true); G->olapsLen++; } delete ovs; fprintf(stderr, "Read_Olaps()-- Loaded "F_U64" overlaps -- "F_U64" normal and "F_U64" innie.\n", G->olapsLen, numNormal, numInnie); } canu-1.3/src/overlapErrorAdjustment/correctOverlaps-Redo_Olaps.C000066400000000000000000000366321272335422300251360ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAY-14 to 2015-JUN-03 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-FEB-25 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "correctOverlaps.H" #include "correctionOutput.H" #include "AS_UTL_reverseComplement.H" void correctRead(uint32 curID, char *fseq, uint32 &fseqLen, Adjust_t *fadj, uint32 &fadjLen, char *oseq, uint32 oseqLen, Correction_Output_t *C, uint64 &Cpos, uint64 Clen, uint64 *changes=NULL); int32 Prefix_Edit_Dist(char *A, int32 m, char *T, int32 n, int32 Error_Limit, int32 &A_End, int32 &T_End, bool &Match_To_End, pedWorkArea_t *ped); #define DISPLAY_WIDTH 250 // Show (to stdout ) the alignment encoded in delta [0 .. (deltaLen - 1)] // between strings a [0 .. (a_len - 1)] and b [0 .. (b_len - 1)] . static void Display_Alignment(char *a, int32 aLen, char *b, int32 bLen, int32 *delta, int32 deltaLen) { int32 i = 0; int32 j = 0; char *top = new char [32 * 1024]; int32 topLen = 0; char *bot = new char [32 * 1024]; int32 botLen = 0; for (int32 k = 0; k < deltaLen; k++) { for (int32 m = 1; m < abs(delta[k]); m++) { top[topLen++] = a[i++]; j++; } if (delta[k] < 0) { top[topLen++] = '-'; j++; } else { top[topLen++] = a[i++]; } } while (i < aLen && j < bLen) { top[topLen++] = a[i++]; j++; } top[topLen] = '\0'; i = j = 0; for (int32 k = 0; k < deltaLen; k++) { for (int32 m = 1; m < abs(delta[k]); m++) { bot[botLen++] = b[j++]; i++; } if (delta[k] > 0) { bot[botLen++] = '-'; i++; } else { bot[botLen++] = b[j++]; } } while (j < bLen && i < aLen) { bot[botLen++] = b[j++]; i++; } bot[botLen] = '\0'; for (i = 0; i < topLen || i < botLen; i += DISPLAY_WIDTH) { putc('\n', stderr); fprintf(stderr, "A: "); for (j = 0; j < DISPLAY_WIDTH && i + j < topLen; j++) putc(top[i + j], stderr); putc('\n', stderr); fprintf(stderr, "B: "); for (j = 0; j < DISPLAY_WIDTH && i + j < botLen; j++) putc(bot[i + j], stderr); putc('\n', stderr); fprintf(stderr, " "); for (j = 0; j < DISPLAY_WIDTH && i + j < botLen && i + j < topLen; j++) if (top[i + j] != ' ' && bot[i + j] != ' ' && top[i + j] != bot[i + j]) putc('^', stderr); else putc(' ', stderr); putc('\n', stderr); } delete [] top; delete [] bot; } // Set hanging offset values for reversed fragment in // rev_adj[0 .. (adj_ct - 1)] based on corresponding forward // values in fadj[0 .. (adj_ct - 1)]. frag_len is the length // of the fragment. static void Make_Rev_Adjust(Adjust_t *radj, Adjust_t *fadj, int32 adj_ct, int32 frag_len) { if (adj_ct == 0) return; int32 i = 0; int32 j = 0; int32 prev = 0; for (i=adj_ct-1; i>0; i--) { if (fadj[i].adjust == fadj[i-1].adjust + 1) { radj[j].adjpos = 2 + frag_len - fadj[i].adjpos; radj[j].adjust = prev + 1; prev = radj[j].adjust; } else if (fadj[i].adjust == fadj[i-1].adjust - 1) { radj[j].adjpos = 3 + frag_len - fadj[i].adjpos; radj[j].adjust = prev - 1; prev = radj[j].adjust; } else { fprintf(stderr, "ERROR: Bad adjustment value. i = %d adj_ct = %d adjust[i] = %d adjust[i-1] = %d\n", i, adj_ct, fadj[i].adjust, fadj[i-1].adjust); assert(0); } j++; } assert(i == 0); if (fadj[i].adjust == 1) { radj[j].adjpos = 2 + frag_len - fadj[i].adjpos; radj[j].adjust = prev + 1; } else if (fadj[i].adjust == -1) { radj[j].adjpos = 3 + frag_len - fadj[i].adjpos; radj[j].adjust = prev - 1; } else { fprintf(stderr, "ERROR: Bad adjustment value. i = %d adj_ct = %d adjust[i] = %d\n", i, adj_ct, fadj[i].adjust); assert(0); } assert(j+1 == adj_ct); } // Return the adjusted value of hang based on // adjust[0 .. (adjust_ct - 1)] . static int32 Hang_Adjust(int32 hang, Adjust_t *adjust, int32 adjust_ct) { int32 delta = 0; assert(hang >= 0); // Replacing second test >= with just > didn't change anything. Both had 14 fails. for (int32 i=0; (i < adjust_ct) && (hang >= adjust[i].adjpos); i++) { //if (delta != adjust[i].adjust) // fprintf(stderr, "hang_adjust i=%d adjust_ct=%d adjust=%d pos=%d\n", i, adjust_ct, adjust[i].adjust, adjust[i].adjpos); delta = adjust[i].adjust; } //fprintf(stderr, "hang adjust delta %d\n", delta); return(hang + delta); } // Read old fragments in gkpStore and choose the ones that // have overlaps with fragments in Frag. Recompute the // overlaps, using fragment corrections and output the revised error. void Redo_Olaps(coParameters *G, gkStore *gkpStore) { // Figure out the range of B reads we care about. We probably could just loop over every read in // the store with minimal penalty. uint64 thisOvl = 0; uint64 lastOvl = G->olapsLen - 1; uint32 loBid = G->olaps[thisOvl].b_iid; uint32 hiBid = G->olaps[lastOvl].b_iid; // Open all the corrections. memoryMappedFile *Cfile = new memoryMappedFile(G->correctionsName); Correction_Output_t *C = (Correction_Output_t *)Cfile->get(); uint64 Cpos = 0; uint64 Clen = Cfile->length() / sizeof(Correction_Output_t); // Allocate some temporary work space for the forward and reverse corrected B reads. fprintf(stderr, "--Allocate "F_U64" MB for fseq and rseq.\n", (2 * sizeof(char) * 2 * (AS_MAX_READLEN + 1)) >> 20); char *fseq = new char [AS_MAX_READLEN + 1 + AS_MAX_READLEN + 1]; uint32 fseqLen = 0; char *rseq = new char [AS_MAX_READLEN + 1 + AS_MAX_READLEN + 1]; uint32 rseqLen = 0; fprintf(stderr, "--Allocate "F_U64" MB for fadj and radj.\n", (2 * sizeof(Adjust_t) * (AS_MAX_READLEN + 1)) >> 20); Adjust_t *fadj = new Adjust_t [AS_MAX_READLEN + 1]; Adjust_t *radj = new Adjust_t [AS_MAX_READLEN + 1]; uint32 fadjLen = 0; // radj is the same length fprintf(stderr, "--Allocate "F_U64" MB for pedWorkArea_t.\n", sizeof(pedWorkArea_t) >> 20); gkReadData *readData = new gkReadData; pedWorkArea_t *ped = new pedWorkArea_t; uint64 Total_Alignments_Ct = 0; uint64 Failed_Alignments_Ct = 0; uint64 Failed_Alignments_Both_Ct = 0; uint64 Failed_Alignments_End_Ct = 0; uint64 Failed_Alignments_Length_Ct = 0; uint32 rhaFail = 0; uint32 rhaPass = 0; uint64 olapsFwd = 0; uint64 olapsRev = 0; ped->initialize(G, G->errorRate); // Process overlaps. Loop over the B reads, and recompute each overlap. for (uint32 curID=loBid; curID<=hiBid; curID++) { if (((curID - loBid) % 1024) == 0) fprintf(stderr, "Recomputing overlaps - %9u - %9u - %9u\r", loBid, curID, hiBid); if (curID < G->olaps[thisOvl].b_iid) continue; gkRead *read = gkpStore->gkStore_getRead(curID); gkpStore->gkStore_loadReadData(read, readData); // Apply corrections to the B read (also converts to lower case, reverses it, etc) //fprintf(stderr, "Correcting B read %u at Cpos=%u\n", curID, Cpos); fseqLen = 0; rseqLen = 0; fadjLen = 0; correctRead(curID, fseq, fseqLen, fadj, fadjLen, readData->gkReadData_getSequence(), read->gkRead_sequenceLength(), C, Cpos, Clen); // Create copies of the sequence for forward and reverse. There isn't a need for the forward copy (except that // we mutate it with corrections), and the reverse copy could be deferred until it is needed. memcpy(rseq, fseq, sizeof(char) * (fseqLen + 1)); reverseComplementSequence(rseq, fseqLen); Make_Rev_Adjust(radj, fadj, fadjLen, fseqLen); // Recompute alignments for all overlaps involving the B read. for (; ((thisOvl <= lastOvl) && (G->olaps[thisOvl].b_iid == curID)); thisOvl++) { Olap_Info_t *olap = G->olaps + thisOvl; //fprintf(stderr, "processing overlap %u - %u\n", olap->a_iid, olap->b_iid); // Find the A segment. It's always forward. It's already been corrected. char *a_part = G->reads[olap->a_iid - G->bgnID].bases; if (olap->a_hang > 0) { int32 ha = Hang_Adjust(olap->a_hang, G->reads[olap->a_iid - G->bgnID].adjusts, G->reads[olap->a_iid - G->bgnID].adjustsLen); a_part += ha; //fprintf(stderr, "offset a_part by ha=%d\n", ha); } // Find the B segment. char *b_part = (olap->normal == true) ? fseq : rseq; //if (olap->normal == true) // fprintf(stderr, "b_part = fseq %40.40s\n", fseq); //else // fprintf(stderr, "b_part = rseq %40.40s\n", rseq); if (olap->normal == true) olapsFwd++; else olapsRev++; bool rha=false; if (olap->a_hang < 0) { int32 ha = (olap->normal == true) ? Hang_Adjust(-olap->a_hang, fadj, fadjLen) : Hang_Adjust(-olap->a_hang, radj, fadjLen); b_part += ha; //fprintf(stderr, "offset b_part by ha=%d normal=%d\n", ha, olap->normal); rha=true; } // Compute the alignment. int32 a_part_len = strlen(a_part); int32 b_part_len = strlen(b_part); int32 olap_len = min(a_part_len, b_part_len); int32 a_end = 0; int32 b_end = 0; bool match_to_end = false; //fprintf(stderr, ">A\n%s\n", a_part); //fprintf(stderr, ">B\n%s\n", b_part); int32 errors = Prefix_Edit_Dist(a_part, a_part_len, b_part, b_part_len, G->Error_Bound[olap_len], a_end, b_end, match_to_end, ped); // ped->delta isn't used. // ?? These both occur, but the first is much much more common. if ((ped->deltaLen > 0) && (ped->delta[0] == 1) && (0 < G->olaps[thisOvl].a_hang)) { int32 stop = min(ped->deltaLen, (int32)G->olaps[thisOvl].a_hang); // a_hang is int32:31! int32 i = 0; for (i=0; (i < stop) && (ped->delta[i] == 1); i++) ; //fprintf(stderr, "RESET 1 i=%d delta=%d\n", i, ped->delta[i]); assert((i == stop) || (ped->delta[i] != -1)); ped->deltaLen -= i; memmove(ped->delta, ped->delta + i, ped->deltaLen * sizeof (int)); a_part += i; a_end -= i; a_part_len -= i; errors -= i; } else if ((ped->deltaLen > 0) && (ped->delta[0] == -1) && (G->olaps[thisOvl].a_hang < 0)) { int32 stop = min(ped->deltaLen, - G->olaps[thisOvl].a_hang); int32 i = 0; for (i=0; (i < stop) && (ped->delta[i] == -1); i++) ; //fprintf(stderr, "RESET 2 i=%d delta=%d\n", i, ped->delta[i]); assert((i == stop) || (ped->delta[i] != 1)); ped->deltaLen -= i; memmove(ped->delta, ped->delta + i, ped->deltaLen * sizeof (int)); b_part += i; b_end -= i; b_part_len -= i; errors -= i; } Total_Alignments_Ct++; int32 olapLen = min(a_end, b_end); if ((match_to_end == false) && (olapLen <= 0)) Failed_Alignments_Both_Ct++; if (match_to_end == false) Failed_Alignments_End_Ct++; if (olapLen <= 0) Failed_Alignments_Length_Ct++; if ((match_to_end == false) || (olapLen <= 0)) { Failed_Alignments_Ct++; #if 0 // I can't find any patterns in these errors. I thought that it was caused by the corrections, but I // found a case where no corrections were made and the alignment still failed. Perhaps it is differences // in the alignment code (the forward vs reverse prefix distance in overlapper vs only the forward here)? fprintf(stderr, "Redo_Olaps()--\n"); fprintf(stderr, "Redo_Olaps()--\n"); fprintf(stderr, "Redo_Olaps()-- Bad alignment errors %d a_end %d b_end %d match_to_end %d olapLen %d\n", errors, a_end, b_end, match_to_end, olapLen); fprintf(stderr, "Redo_Olaps()-- Overlap a_hang %d b_hang %d innie %d\n", olap->a_hang, olap->b_hang, olap->innie); fprintf(stderr, "Redo_Olaps()-- Reads a_id %u a_length %d b_id %u b_length %d\n", G->olaps[thisOvl].a_iid, G->reads[ G->olaps[thisOvl].a_iid ].basesLen, G->olaps[thisOvl].b_iid, G->reads[ G->olaps[thisOvl].b_iid ].basesLen); fprintf(stderr, "Redo_Olaps()-- A %s\n", a_part); fprintf(stderr, "Redo_Olaps()-- B %s\n", b_part); Display_Alignment(a_part, a_part_len, b_part, b_part_len, ped->delta, ped->deltaLen); fprintf(stderr, "\n"); #endif if (rha) rhaFail++; continue; } if (rha) rhaPass++; G->olaps[thisOvl].evalue = AS_OVS_encodeEvalue((double)errors / olapLen); //fprintf(stderr, "REDO - errors = %u / olapLep = %u -- %f\n", errors, olapLen, AS_OVS_decodeEvalue(G->olaps[thisOvl].evalue)); } } fprintf(stderr, "\n"); delete ped; delete readData; delete [] radj; delete [] fadj; delete [] rseq; delete [] fseq; delete Cfile; fprintf(stderr, "-- Release bases, adjusts and reads.\n"); delete [] G->bases; G->bases = NULL; delete [] G->adjusts; G->adjusts = NULL; delete [] G->reads; G->reads = NULL; fprintf(stderr, "Olaps Fwd "F_U64"\n", olapsFwd); fprintf(stderr, "Olaps Rev "F_U64"\n", olapsRev); fprintf(stderr, "Total: "F_U64"\n", Total_Alignments_Ct); fprintf(stderr, "Failed: "F_U64" (both)\n", Failed_Alignments_Both_Ct); fprintf(stderr, "Failed: "F_U64" (either)\n", Failed_Alignments_Ct); fprintf(stderr, "Failed: "F_U64" (match to end)\n", Failed_Alignments_End_Ct); fprintf(stderr, "Failed: "F_U64" (negative length)\n", Failed_Alignments_Length_Ct); fprintf(stderr, "rhaFail %u rhaPass %u\n", rhaFail, rhaPass); } canu-1.3/src/overlapErrorAdjustment/correctOverlaps.C000066400000000000000000000156071272335422300231100ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2015-JUN-16 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-MAR-30 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "correctOverlaps.H" #include "Binomial_Bound.H" void Read_Olaps(coParameters *G, gkStore *gkpStore); void Correct_Frags(coParameters *G, gkStore *gkpStore); void Redo_Olaps(coParameters *G, gkStore *gkpStore); int main(int argc, char **argv) { coParameters *G = new coParameters(); argc = AS_configure(argc, argv); int arg = 1; int err = 0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { G->gkpStorePath = argv[++arg]; } else if (strcmp(argv[arg], "-R") == 0) { G->bgnID = atoi(argv[++arg]); G->endID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-O") == 0) { // -F? -S Olap_Path G->ovlStorePath = argv[++arg]; } else if (strcmp(argv[arg], "-e") == 0) { G->errorRate = atof(argv[++arg]); } else if (strcmp(argv[arg], "-l") == 0) { G->minOverlap = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-c") == 0) { // For 'corrections' file input G->correctionsName = argv[++arg]; } else if (strcmp(argv[arg], "-o") == 0) { // For 'erates' output G->eratesName = argv[++arg]; } else if (strcmp(argv[arg], "-t") == 0) { // But we're not threaded! G->numThreads = atoi(argv[++arg]); } else { err++; } arg++; } if (G->gkpStorePath == NULL) fprintf(stderr, "ERROR: no input gatekeeper store (-G) supplied.\n"), err++; if (G->ovlStorePath == NULL) fprintf(stderr, "ERROR: no input overlap store (-O) supplied.\n"), err++; if (G->correctionsName == NULL) fprintf(stderr, "ERROR: no input read corrections file (-c) supplied.\n"), err++; if (G->eratesName == NULL) fprintf(stderr, "ERROR: no output erates file (-o) supplied.\n"), err++; if (err) { fprintf(stderr, "USAGE: %s [-d ] [-o ] [-q ]\n", argv[0]); fprintf(stderr, " [-x ] [-F OlapFile] [-S OlapStore]\n"); fprintf(stderr, " [-c ] [-e \n"); fprintf(stderr, " \n"); fprintf(stderr, "\n"); fprintf(stderr, "Recalculates overlaps for frags .. in\n"); fprintf(stderr, " using corrections in \n"); fprintf(stderr, "\n"); fprintf(stderr, "Options:\n"); fprintf(stderr, "-e specifies binary file to dump corrected erates to\n"); fprintf(stderr, " for later updating of olap store by update-erates \n"); fprintf(stderr, "-F specify file of sorted overlaps to use (in the format\n"); fprintf(stderr, " produced by get-olaps\n"); fprintf(stderr, "-o specifies name of file to which OVL messages go\n"); fprintf(stderr, "-q overlaps less than this error rate are\n"); fprintf(stderr, " automatically output\n"); fprintf(stderr, "-S specify the binary overlap store containing overlaps to use\n"); exit(1); } //fprintf (stderr, "Quality Threshold = %.2f%%\n", 100.0 * Quality_Threshold); // // Initialize Globals // fprintf(stderr, "Initializing.\n"); double MAX_ERRORS = 1 + (uint32)(G->errorRate * AS_MAX_READLEN); Initialize_Match_Limit(G->Edit_Match_Limit, G->errorRate, MAX_ERRORS); for (int32 i=0; i <= AS_MAX_READLEN; i++) G->Error_Bound[i] = (int)ceil(i * G->errorRate); // // // fprintf(stderr, "Opening gkpStore '%s'.\n", G->gkpStorePath); gkStore *gkpStore = gkStore::gkStore_open(G->gkpStorePath); if (G->bgnID < 1) G->bgnID = 1; if (gkpStore->gkStore_getNumReads() < G->endID) G->endID = gkpStore->gkStore_getNumReads(); // Load the reads for the overlaps we are going to be correcting, and apply corrections to them fprintf(stderr, "Correcting reads "F_U32" to "F_U32".\n", G->bgnID, G->endID); Correct_Frags(G, gkpStore); // Load overlaps we're going to correct fprintf(stderr, "Loading overlaps.\n"); Read_Olaps(G, gkpStore); // Now sort them on the B iid. fprintf(stderr, "Sorting overlaps.\n"); #ifdef _GLIBCXX_PARALLEL __gnu_sequential::sort(G->olaps, G->olaps + G->olapsLen, Olap_Info_t_by_bID()); #else sort(G->olaps, G->olaps + G->olapsLen, Olap_Info_t_by_bID()); #endif // Recompute overlaps fprintf(stderr, "Recomputing overlaps.\n"); Redo_Olaps(G, gkpStore); gkpStore->gkStore_close(); gkpStore = NULL; // Sort the overlaps back into the original order fprintf(stderr, "Sorting overlaps.\n"); #ifdef _GLIBCXX_PARALLEL __gnu_sequential::sort(G->olaps, G->olaps + G->olapsLen, Olap_Info_t_by_Order()); #else sort(G->olaps, G->olaps + G->olapsLen, Olap_Info_t_by_Order()); #endif // Dump the new erates fprintf (stderr, "Saving corrected error rates to file %s\n", G->eratesName); { errno = 0; FILE *fp = fopen(G->eratesName, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", G->eratesName, strerror(errno)), exit(1); AS_UTL_safeWrite(fp, &G->bgnID, "loid", sizeof(int32), 1); AS_UTL_safeWrite(fp, &G->endID, "hiid", sizeof(int32), 1); AS_UTL_safeWrite(fp, &G->olapsLen, "num", sizeof(uint64), 1); fprintf(stderr, "--Allocate "F_U64" MB for output error rates.\n", (sizeof(uint16) * G->olapsLen) >> 20); uint16 *evalue = new uint16 [G->olapsLen]; for (int32 i=0; iolapsLen; i++) evalue[i] = G->olaps[i].evalue; AS_UTL_safeWrite(fp, evalue, "evalue", sizeof(uint16), G->olapsLen); delete [] evalue; fclose(fp); } // Finished. //fprintf (stderr, "%d/%d failed/total alignments (%.1f%%)\n", // Failed_Alignments_Ct, Total_Alignments_Ct, // Total_Alignments_Ct == 0 ? 0.0 : (100.0 * Failed_Alignments_Ct) / Total_Alignments_Ct); delete G; fprintf(stderr, "DONE.\n"); exit(0); } canu-1.3/src/overlapErrorAdjustment/correctOverlaps.H000066400000000000000000000165771272335422300231240ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAY-02 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include #include "gkStore.H" #include "ovStore.H" #include using namespace std; #define Sign(a) ( ((a) > 0) - ((a) < 0) ) // Value to add for a match in finding branch points // 1.20 was the calculated value for 6% vs 35% error discrimination // Converting to integers didn't make it faster #define BRANCH_PT_MATCH_VALUE 0.272 // Value to add for a mismatch in finding branch points // -2.19 was the calculated value for 6% vs 35% error discrimination // Converting to integers didn't make it faster #define BRANCH_PT_ERROR_VALUE -0.728 // Default value for End_Exclude_Len #define DEFAULT_END_EXCLUDE_LEN 3 // Default value for bases on each side of SNP to vote for change #define DEFAULT_HALF_LEN 4 // Default value for Kmer_Len #define DEFAULT_KMER_LEN 9 // Default value for Quality_Threshold #define DEFAULT_QUALITY_THRESHOLD 0.015 // Probability limit to "band" edit-distance calculation // Determines NORMAL_DISTRIB_THOLD #define EDIT_DIST_PROB_BOUND 1e-4 // The number of errors that are ignored in setting probability // bound for terminating alignment extensions in edit distance // calculations #define ERRORS_FOR_FREE 1 // Most bytes allowed in line of fasta file #define MAX_FASTA_LINE 2048 // Longest name allowed for a file in the overlap store #define MAX_FILENAME_LEN 1000 // Most errors in any edit distance computation // 0.40 // KNOWN ONLY AT RUN TIME //#define MAX_ERRORS (1 + (int) (AS_OVL_ERROR_RATE * AS_MAX_READLEN)) // Factor by which to grow memory in olap array when reading it #define EXPANSION_FACTOR 1.4 // Branch points must be at least this many bases from the // end of the fragment to be reported #define MIN_BRANCH_END_DIST 20 // Branch point tails must fall off from the max by at least // this rate #define MIN_BRANCH_TAIL_SLOPE 0.20 // Determined by EDIT_DIST_PROB_BOUND #define NORMAL_DISTRIB_THOLD 3.62 struct Adjust_t { int32 adjpos; int32 adjust; }; class Frag_Info_t { public: Frag_Info_t() { bases = NULL; basesLen = 0; adjusts = NULL; adjustsLen = 0; keep_left = false; keep_right = false; }; ~Frag_Info_t() { }; char *bases; Adjust_t *adjusts; uint32 basesLen; uint32 adjustsLen; uint32 keep_right : 1; // I think these are unused. uint32 keep_left : 1; // If so, we get back 8 bytes. If not, redo Correct_Frags to use a temporary, and use 31 bits for the two lengths. }; class Olap_Info_t { public: Olap_Info_t() {}; ~Olap_Info_t() {}; uint32 a_iid; uint32 b_iid; int64 a_hang : 31; int64 b_hang : 31; uint64 innie : 1; // was 'orient' with choice INNIE=0 or NORMAL=1 uint64 normal : 1; // so 'normal' always != 'innie' uint64 order; uint32 evalue; }; // Sort by increasing b_iid. // // It is possible, but unlikely, to have two overlaps to the same pair of reads, // if we overlap a5'-b3' and a3'-b5'. I think. // class Olap_Info_t_by_bID { public: inline bool operator()(const Olap_Info_t &a, const Olap_Info_t &b) { if (a.b_iid < b.b_iid) return(true); if (a.b_iid > b.b_iid) return(false); if (a.a_iid < b.a_iid) return(true); if (a.a_iid > b.a_iid) return(false); return(a.innie != b.innie); }; }; class Olap_Info_t_by_Order { public: inline bool operator()(const Olap_Info_t &a, const Olap_Info_t &b) { return(a.order < b.order); }; }; class coParameters; class pedWorkArea_t { public: pedWorkArea_t() { G = NULL; memset(delta, 0, sizeof(int32) * AS_MAX_READLEN); memset(deltaStack, 0, sizeof(int32) * AS_MAX_READLEN); deltaLen = 0; Edit_Array_Lazy = NULL; }; ~pedWorkArea_t() { for (uint32 xx=0; xx < alloc.size(); xx++) delete [] alloc[xx]; delete [] Edit_Array_Lazy; }; void initialize(coParameters *G_, double errorRate) { G = G_; fprintf(stderr, "-- Allocate "F_U64" MB for Edit_Array pointers.\n", (sizeof(int32 *) * Edit_Array_Max) >> 20); Edit_Array_Max = 1 + (uint32)(errorRate * AS_MAX_READLEN); Edit_Array_Lazy = new int32 * [Edit_Array_Max]; memset(Edit_Array_Lazy, 0, sizeof(int32 *) * Edit_Array_Max); }; public: coParameters *G; int32 delta[AS_MAX_READLEN]; // Only need ERATE * READLEN int32 deltaStack[AS_MAX_READLEN]; int32 deltaLen; vector alloc; // Allocated blocks, don't use directly. int32 **Edit_Array_Lazy; // Doled out space. int32 Edit_Array_Max; // Former MAX_ERRORS }; class coParameters { public: coParameters() { gkpStorePath = NULL; ovlStorePath = NULL; // Input read corrections, output overlap corrections correctionsName = NULL; eratesName = NULL; // Range of IDs to process bgnID = 0; endID = UINT32_MAX; bases = NULL; basesLen = 0; adjusts = NULL; adjustsLen = 0; reads = NULL; readsLen = 0; olaps = NULL; olapsLen = 0; numThreads = 1; errorRate = 0.06; minOverlap = 0; }; ~coParameters() { delete [] bases; delete [] adjusts; delete [] reads; delete [] olaps; }; // Paths to stores char *gkpStorePath; char *ovlStorePath; // Input read corrections, output overlap corrections char *correctionsName; char *eratesName; // Range of IDs to process uint32 bgnID; uint32 endID; char *bases; uint64 basesLen; Adjust_t *adjusts; uint64 adjustsLen; Frag_Info_t *reads; // These are relative to bgnID! uint32 readsLen; // Number of fragments being corrected Olap_Info_t *olaps; uint64 olapsLen; // Number of overlaps being used uint32 numThreads; // Only one thread supported. double errorRate; uint32 minOverlap; pedWorkArea_t ped; // Globals // This array[e] is the minimum value of Edit_Array[e][d] // to be worth pursuing in edit-distance computations between guides // (only MAX_ERRORS needed) int Edit_Match_Limit[AS_MAX_READLEN+1]; // This array[i] is the maximum number of errors allowed // in a match between sequences of length i , which is // i * MAXERROR_RATE . int Error_Bound[AS_MAX_READLEN + 1]; }; canu-1.3/src/overlapErrorAdjustment/correctOverlaps.mk000066400000000000000000000012761272335422300233320ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := correctOverlaps SOURCES := correctOverlaps.C \ correctOverlaps-Correct_Frags.C \ correctOverlaps-Read_Olaps.C \ correctOverlaps-Redo_Olaps.C \ correctOverlaps-Prefix_Edit_Distance.C SRC_INCDIRS := .. ../AS_UTL ../stores ../overlapInCore/liboverlap TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/overlapErrorAdjustment/correctionOutput.H000066400000000000000000000024601272335422300233210ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ // Definitions for our exportable data. enum Vote_Value_t { IDENT, // Just an iid in this record. DELETE, A_SUBST, C_SUBST, G_SUBST, T_SUBST, // DON'T rearrange this! Code depends on the ordering. A_INSERT, C_INSERT, G_INSERT, T_INSERT, NO_VOTE, EXTENSION }; struct Correction_Output_t { uint32 keep_left : 1; // set true if left overlap degree is low uint32 keep_right : 1; // set true if right overlap degree is low uint32 type : 4; // Vote_Value_t uint32 pos : 26; // uint32 readID; }; canu-1.3/src/overlapErrorAdjustment/findErrors-Analyze_Alignment.C000066400000000000000000000232161272335422300254420ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz on 2015-JUN-18 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Sergey Koren beginning on 2016-MAR-22 * are a 'United States Government Work', and * are released in the public domain * * Brian P. Walenz beginning on 2016-MAY-18 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "findErrors.H" // Add vote val to G.reads[sub] at sequence position p static void Cast_Vote(feParameters *G, Vote_Value_t val, int32 pos, int32 sub) { int32 v=0; switch (val) { case DELETE: if (G->reads[sub].vote[pos].deletes < MAX_VOTE) v = ++G->reads[sub].vote[pos].deletes; break; case A_SUBST: if (G->reads[sub].vote[pos].a_subst < MAX_VOTE) v = ++G->reads[sub].vote[pos].a_subst; break; case C_SUBST: if (G->reads[sub].vote[pos].c_subst < MAX_VOTE) v = ++G->reads[sub].vote[pos].c_subst; break; case G_SUBST: if (G->reads[sub].vote[pos].g_subst < MAX_VOTE) v = ++G->reads[sub].vote[pos].g_subst; break; case T_SUBST: if (G->reads[sub].vote[pos].t_subst < MAX_VOTE) v = ++G->reads[sub].vote[pos].t_subst; break; case A_INSERT: if (G->reads[sub].vote[pos].a_insert < MAX_VOTE) v = ++G->reads[sub].vote[pos].a_insert; break; case C_INSERT: if (G->reads[sub].vote[pos].c_insert < MAX_VOTE) v = ++G->reads[sub].vote[pos].c_insert; break; case G_INSERT: if (G->reads[sub].vote[pos].g_insert < MAX_VOTE) v = ++G->reads[sub].vote[pos].g_insert; break; case T_INSERT: if (G->reads[sub].vote[pos].t_insert < MAX_VOTE) v = ++G->reads[sub].vote[pos].t_insert; break; case NO_VOTE: break; default : fprintf(stderr, "ERROR: Illegal vote type\n"); break; } // Largely useless, just too much output. //fprintf(stderr, "Cast_Vote()-- sub %d at %d vote %d\n", sub, p, val); } // Return the substitution vote corresponding to Ch . static Vote_Value_t Matching_Vote(char ch) { switch (ch) { case 'a': return(A_SUBST); break; case 'c': return(C_SUBST); break; case 'g': return(G_SUBST); break; case 't': return(T_SUBST); break; } fprintf(stderr, "Matching_Vote()-- invalid letter '%c'\n", ch); return(NO_VOTE); } // Analyze the delta-encoded alignment in delta[0 .. (deltaLen - 1)] // between a_part and b_part and store the resulting votes // about the a sequence in G->reads[sub]. The alignment starts // a_offset bytes in from the start of the a sequence in G->reads[sub] . // a_len and b_len are the lengths of the prefixes of a_part and // b_part , resp., that align. void Analyze_Alignment(Thread_Work_Area_t *wa, char *a_part, int32 a_len, int32 a_offset, char *b_part, int32 b_len, int32 sub) { assert(a_len >= 0); assert(b_len >= 0); int32 ct = 0; // Necessary?? //memset(wa->globalvote, 0, sizeof(Vote_t) * AS_MAX_READLEN); wa->globalvote[ct].frag_sub = -1; wa->globalvote[ct].align_sub = -1; wa->globalvote[ct].vote_val = A_SUBST; // Dummy value ct++; int32 i = 0; int32 j = 0; int32 p = 0; for (int32 k=0; kped.deltaLen; k++) { //fprintf(stderr, "k=%d deltalen=%d i=%d our of %d j=%d out of %d\n", k, wa->ped.deltaLen, i, a_len, j, b_len); // Add delta[k] matches or mismatches for (int32 m=1; mped.delta[k]); m++) { if (a_part[i] != b_part[j]) { wa->globalvote[ct].frag_sub = i; wa->globalvote[ct].align_sub = p; switch (b_part[j]) { case 'a': wa->globalvote[ct].vote_val = A_SUBST; break; case 'c': wa->globalvote[ct].vote_val = C_SUBST; break; case 'g': wa->globalvote[ct].vote_val = G_SUBST; break; case 't': wa->globalvote[ct].vote_val = T_SUBST; break; default : fprintf(stderr, "ERROR:[1] Bad sequence '%c' 0x%02x)\n", b_part[j], b_part[j]); assert(0); } ct++; } i++; //assert(i <= a_len); j++; //assert(j <= b_len); p++; } // If a negative delta, insert a base. if (wa->ped.delta[k] < 0) { wa->globalvote[ct].frag_sub = i - 1; wa->globalvote[ct].align_sub = p; //fprintf(stderr, "INSERT %c at %d #%d\n", b_part[j], i-1, p); switch (b_part[j]) { case 'a': wa->globalvote[ct].vote_val = A_INSERT; break; case 'c': wa->globalvote[ct].vote_val = C_INSERT; break; case 'g': wa->globalvote[ct].vote_val = G_INSERT; break; case 't': wa->globalvote[ct].vote_val = T_INSERT; break; default : fprintf(stderr, "ERROR:[2] Bad sequence '%c' 0x%02x)\n", b_part[j], b_part[j]); assert(0); } ct++; j++; //assert(j <= b_len); p++; } // If a positive deta, delete the base. if (wa->ped.delta[k] > 0) { wa->globalvote[ct].frag_sub = i; wa->globalvote[ct].align_sub = p; wa->globalvote[ct].vote_val = DELETE; //fprintf(stderr, "DELETE %c at %d #%d\n", a_part[i], i, p); ct++; i++; assert(i <= a_len); p++; } } // No more deltas. While there is still sequence, add matches or mismatches. //fprintf(stderr, "k=done i=%d our of %d j=%d out of %d\n", i, a_len, j, b_len); while (i < a_len) { //fprintf(stderr, "k=done i=%d our of %d j=%d out of %d\n", i, a_len, j, b_len); if (a_part[i] != b_part[j]) { wa->globalvote[ct].frag_sub = i; wa->globalvote[ct].align_sub = p; switch (b_part[j]) { case 'a': wa->globalvote[ct].vote_val = A_SUBST; break; case 'c': wa->globalvote[ct].vote_val = C_SUBST; break; case 'g': wa->globalvote[ct].vote_val = G_SUBST; break; case 't': wa->globalvote[ct].vote_val = T_SUBST; break; default : fprintf(stderr, "ERROR:[3] Bad sequence '%c' 0x%02x)\n", b_part[j], b_part[j]); assert(0); } ct++; } i++; //assert(i <= a_len); // Guaranteed, we're looping on this j++; //assert(j <= b_len); p++; } wa->globalvote[ct].frag_sub = i; wa->globalvote[ct].align_sub = p; // For each identified change, add votes for some region around the change. // // This is adding extra votes if the distance between two errors is larger than a kmer. // Not sure why there are no 'matching bases' in this region. // // X == changes, mismatch or indel // // ------- <- confirmed count added // ----- <- no_insert count added // matching-bases} X 1 2 3 1 2 3 4 3 2 1 X {matching-bases // ----- ----- // match match // votes votes // for (int32 i=1; i<=ct; i++) { int32 prev_match = wa->globalvote[i].align_sub - wa->globalvote[i - 1].align_sub - 1; int32 p_lo = (i == 1 ? 0 : wa->G->End_Exclude_Len); int32 p_hi = (i == ct ? prev_match : prev_match - wa->G->End_Exclude_Len); // If distance to previous match is bigger than 'kmer' size, make a new vote. if (prev_match >= wa->G->Kmer_Len) { for (int32 p=0; pG, Matching_Vote(a_part[wa->globalvote[i-1].frag_sub + p + 1]), a_offset + wa->globalvote[i-1].frag_sub + p + 1, sub); for (int32 p=p_lo; pglobalvote[i-1].frag_sub + p + 1; if (wa->G->reads[sub].vote[k].confirmed < MAX_VOTE) wa->G->reads[sub].vote[k].confirmed++; if ((p < p_hi - 1) && (wa->G->reads[sub].vote[k].no_insert < MAX_VOTE)) wa->G->reads[sub].vote[k].no_insert++; } for (int32 p=p_hi; pG, Matching_Vote(a_part[wa->globalvote[i-1].frag_sub + p + 1]), a_offset + wa->globalvote[i-1].frag_sub + p + 1, sub); } // Don't allow consecutive inserts. If we aren't the last change, and there is non-adjacent // previous (or this and the previous votes are not insertions), do another vote. if ((i < ct) && ((prev_match > 0) || (wa->globalvote[i-1].vote_val <= T_SUBST) || (wa->globalvote[i ].vote_val <= T_SUBST))) { int32 next_match = wa->globalvote[i + 1].align_sub - wa->globalvote[i].align_sub - 1; // if our vote is outside of the bounds (meaning we have gaps at the start or end of the alignment), skip the vote if (a_offset + wa->globalvote[i].frag_sub < 0 || a_offset + wa->globalvote[i].frag_sub >= a_len) { continue; } if (prev_match + next_match >= wa->G->Vote_Qualify_Len) Cast_Vote(wa->G, wa->globalvote[i].vote_val, a_offset + wa->globalvote[i].frag_sub, sub); } } } canu-1.3/src/overlapErrorAdjustment/findErrors-Dump.C000066400000000000000000000050511272335422300227430ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2016-MAY-20 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "findErrors.H" int main(int argc, char **argv) { char *redName = NULL; argc = AS_configure(argc, argv); int arg = 1; int err = 0; while (arg < argc) { if (strcmp(argv[arg], "-r") == 0) { redName = argv[++arg]; } else { fprintf(stderr, "Unknown option '%s'\n", argv[arg]); err++; } arg++; } if (redName == NULL) err++; if (err > 0) { fprintf(stderr, "usage: %s -r file.red\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, "Dumps, as ASCII, the results from findErrors *.red files.\n"); if (redName == NULL) fprintf(stderr, "ERROR: no *.red file (-r) supplied.\n"); exit(1); } char *typeName[13] = { "IDENT", "DELETE", "A_SUBST", "C_SUBST", "G_SUBST", "T_SUBST", "A_INSERT", "C_INSERT", "G_INSERT", "T_INSERT", "NO_VOTE", "EXTENSION", NULL }; memoryMappedFile *Cfile = new memoryMappedFile(redName); Correction_Output_t *C = (Correction_Output_t *)Cfile->get(); uint64 Cpos = 0; uint64 Clen = Cfile->length() / sizeof(Correction_Output_t); for (uint32 ii=0; ii%d\n", G->bgnID + i); for (uint32 j=0; G->reads[i].sequence[j] != '\0'; j++) fprintf(stderr, "%3d: %c conf %3d deletes %3d | subst %3d %3d %3d %3d | no_insert %3d insert %3d %3d %3d %3d\n", j, j >= G->reads[i].clear_len ? toupper (G->reads[i].sequence[j]) : G->reads[i].sequence[j], G->reads[i].vote[j].confirmed, G->reads[i].vote[j].deletes, G->reads[i].vote[j].a_subst, G->reads[i].vote[j].c_subst, G->reads[i].vote[j].g_subst, G->reads[i].vote[j].t_subst, G->reads[i].vote[j].no_insert, G->reads[i].vote[j].a_insert, G->reads[i].vote[j].c_insert, G->reads[i].vote[j].g_insert, G->reads[i].vote[j].t_insert); } void Output_Corrections(feParameters *G) { Correction_Output_t out; errno = 0; FILE *fp = fopen(G->outputFileName, "wb"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", G->outputFileName, strerror(errno)), exit(1); for (uint32 i=0; ireadsLen; i++) { //if (i == 0) // Output_Details(G, i); out.keep_left = (G->reads[i].left_degree < G->Degree_Threshold); out.keep_right = (G->reads[i].right_degree < G->Degree_Threshold); out.type = IDENT; out.pos = 0; out.readID = G->bgnID + i; //fprintf(stderr, "read %d clear_len %d\n", i, G->reads[i].clear_len); AS_UTL_safeWrite(fp, &out, "correction1", sizeof(Correction_Output_t), 1); if (G->reads[i].sequence == NULL) // Deleted fragment continue; for (uint32 j=0; jreads[i].clear_len; j++) { if (G->reads[i].vote[j].confirmed < 2) { Vote_Value_t vote = DELETE; int32 max = G->reads[i].vote[j].deletes; bool is_change = true; if (G->reads[i].vote[j].a_subst > max) { vote = A_SUBST; max = G->reads[i].vote[j].a_subst; is_change = (G->reads[i].sequence[j] != 'a'); } if (G->reads[i].vote[j].c_subst > max) { vote = C_SUBST; max = G->reads[i].vote[j].c_subst; is_change = (G->reads[i].sequence[j] != 'c'); } if (G->reads[i].vote[j].g_subst > max) { vote = G_SUBST; max = G->reads[i].vote[j].g_subst; is_change = (G->reads[i].sequence[j] != 'g'); } if (G->reads[i].vote[j].t_subst > max) { vote = T_SUBST; max = G->reads[i].vote[j].t_subst; is_change = (G->reads[i].sequence[j] != 't'); } int32 haplo_ct = ((G->reads[i].vote[j].deletes >= MIN_HAPLO_OCCURS) + (G->reads[i].vote[j].a_subst >= MIN_HAPLO_OCCURS) + (G->reads[i].vote[j].c_subst >= MIN_HAPLO_OCCURS) + (G->reads[i].vote[j].g_subst >= MIN_HAPLO_OCCURS) + (G->reads[i].vote[j].t_subst >= MIN_HAPLO_OCCURS)); int32 total = (G->reads[i].vote[j].deletes + G->reads[i].vote[j].a_subst + G->reads[i].vote[j].c_subst + G->reads[i].vote[j].g_subst + G->reads[i].vote[j].t_subst); // The original had a gargantuajn if test (five clauses, all had to be true) to decide if a record should be output. // It was negated into many small tests if we should skip the output. // A side effect is that we can abort a little earlier in two cases (and we don't even bother). //fprintf(stderr, "TEST read %d position %d type %d -- ", i, j, vote); // (total > 1) if (total <= 1) { //fprintf(stderr, "FEW total = %d <= 1\n", total); continue; } // (2 * max > total) if (2 * max <= total) { //fprintf(stderr, "WEAK 2*max = %d <= total = %d\n", 2*max, total); continue; } // (is_change == true) if (is_change == false) { //fprintf(stderr, "SAME is_change = %d\n", is_change); continue; } // ((haplo_ct < 2) || (G->Use_Haplo_Ct == false)) if ((haplo_ct >= 2) && (G->Use_Haplo_Ct == true)) { //fprintf(stderr, "HAPLO haplo_ct=%d >= 2 AND Use_Haplo_Ct = %d\n", haplo_ct, G->Use_Haplo_Ct); continue; } // ((G->reads[i].vote[j].confirmed == 0) || // ((G->reads[i].vote[j].confirmed == 1) && (max > 6))) if ((G->reads[i].vote[j].confirmed > 0) && ((G->reads[i].vote[j].confirmed != 1) || (max <= 6))) { //fprintf(stderr, "INDET confirmed = %d max = %d\n", G->reads[i].vote[j].confirmed, max); continue; } // Otherwise, output. out.type = vote; out.pos = j; //fprintf(stderr, "CORRECT!\n"); AS_UTL_safeWrite(fp, &out, "correction2", sizeof(Correction_Output_t), 1); } // confirmed < 2 if (G->reads[i].vote[j].no_insert < 2) { Vote_Value_t ins_vote = A_INSERT; int32 ins_max = G->reads[i].vote[j].a_insert; if (ins_max < G->reads[i].vote[j].c_insert) { ins_vote = C_INSERT; ins_max = G->reads[i].vote[j].c_insert; } if (ins_max < G->reads[i].vote[j].g_insert) { ins_vote = G_INSERT; ins_max = G->reads[i].vote[j].g_insert; } if (ins_max < G->reads[i].vote[j].t_insert) { ins_vote = T_INSERT; ins_max = G->reads[i].vote[j].t_insert; } int32 ins_haplo_ct = ((G->reads[i].vote[j].a_insert >= MIN_HAPLO_OCCURS) + (G->reads[i].vote[j].c_insert >= MIN_HAPLO_OCCURS) + (G->reads[i].vote[j].g_insert >= MIN_HAPLO_OCCURS) + (G->reads[i].vote[j].t_insert >= MIN_HAPLO_OCCURS)); int32 ins_total = (G->reads[i].vote[j].a_insert + G->reads[i].vote[j].c_insert + G->reads[i].vote[j].g_insert + G->reads[i].vote[j].t_insert); //fprintf(stderr, "TEST read %d position %d type %d (insert) -- ", i, j, ins_vote); if (ins_total <= 1) { //fprintf(stderr, "FEW ins_total = %d <= 1\n", ins_total); continue; } if (2 * ins_max >= ins_total) { //fprintf(stderr, "WEAK 2*ins_max = %d <= ins_total = %d\n", 2*ins_max, ins_total); continue; } if ((ins_haplo_ct >= 2) && (G->Use_Haplo_Ct == true)) { //fprintf(stderr, "HAPLO ins_haplo_ct=%d >= 2 AND Use_Haplo_Ct = %d\n", ins_haplo_ct, G->Use_Haplo_Ct); continue; } if ((G->reads[i].vote[j].no_insert > 0) && ((G->reads[i].vote[j].no_insert != 1) || (ins_max <= 6))) { //fprintf(stderr, "INDET no_insert = %d ins_max = %d\n", G->reads[i].vote[j].no_insert, ins_max); continue; } // Otherwise, output. out.type = ins_vote; out.pos = j; //fprintf(stderr, "INSERT!\n"); AS_UTL_safeWrite(fp, &out, "correction3", sizeof(Correction_Output_t), 1); } // insert < 2 } } fclose (fp); } canu-1.3/src/overlapErrorAdjustment/findErrors-Prefix_Edit_Distance.C000066400000000000000000000203011272335422300260450ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "findErrors.H" // Set delta to the entries indicating the insertions/deletions // in the alignment encoded in edit_array ending at position // edit_array[e][d]. row is the position in the first // string where the alignment ended. Set (*delta_len) to // the number of entries in delta . static void Compute_Delta(pedWorkArea_t *WA, int32 e, int32 d, int32 row) { int32 last = row; int32 stackLen = 0; for (int32 k=e; k>0; k--) { int32 from = d; int32 max = 1 + WA->Edit_Array_Lazy[k-1][d]; int32 j = WA->Edit_Array_Lazy[k-1][d-1]; if (j > max) { from = d-1; max = j; } j = 1 + WA->Edit_Array_Lazy[k-1][d+1]; if (j > max) { from = d+1; max = j; } if (from == d-1) { WA->deltaStack[stackLen++] = max - last - 1; d--; last = WA->Edit_Array_Lazy[k-1][from]; } else if (from == d+1) { WA->deltaStack[stackLen++] = last - (max - 1); d++; last = WA->Edit_Array_Lazy[k-1][from]; } } WA->deltaStack[stackLen++] = last + 1; for (int32 k=0, i=stackLen-1; i>0; i--) WA->delta[k++] = abs(WA->deltaStack[i]) * Sign(WA->deltaStack[i-1]); WA->deltaLen = stackLen - 1; } // Allocate another block of 64mb for edits // Needs to be at least: // 52,432 to handle 40% error at 64k overlap // 104,860 to handle 80% error at 64k overlap // 209,718 to handle 40% error at 256k overlap // 419,434 to handle 80% error at 256k overlap // 3,355,446 to handle 40% error at 4m overlap // 6,710,890 to handle 80% error at 4m overlap // Bigger means we can assign more than one Edit_Array[] in one allocation. uint32 EDIT_SPACE_SIZE = 16 * 1024 * 1024; static void Allocate_More_Edit_Space(pedWorkArea_t *WA) { // Determine the last allocated block, and the last assigned block int32 b = 0; // Last edit array assigned int32 e = 0; // Last edit array assigned more space int32 a = WA->alloc.size(); // First unallocated block while (WA->Edit_Array_Lazy[b] != NULL) b++; // Fill in the edit space array. Well, not quite yet. First, decide the minimum size. // // Element [0] can access from [-2] to [2] = 5 elements. // Element [1] can access from [-3] to [3] = 7 elements. // // Element [e] can access from [-2-e] to [2+e] = 5 + e * 2 elements // // So, our offset for this new block needs to put [e][0] at offset... int32 Offset = 2 + b; int32 Del = 6 + b * 2; int32 Size = EDIT_SPACE_SIZE; while (Size < Offset + Del) Size *= 2; // Allocate another block int32 *alloc = new int32 [Size]; WA->alloc.push_back(alloc); // And, now, fill in the edit space array. e = b; while ((Offset + Del < Size) && (e < WA->Edit_Array_Max)) { WA->Edit_Array_Lazy[e++] = alloc + Offset; Offset += Del; Del += 2; } if (e == b) fprintf(stderr, "Allocate_More_Edit_Space()-- ERROR: couldn't allocate enough space for even one more entry! e=%d\n", e); assert(e != b); //fprintf(stderr, "WorkArea %d allocates space %d of size %d for array %d through %d\n", WA->thread_id, a, Size, b, e-1); } // Return the minimum number of changes (inserts, deletes, replacements) // needed to match string A[0 .. (m-1)] with a prefix of string // T[0 .. (n-1)] if it's not more than Error_Limit . // // Put delta description of alignment in WA->delta and set // WA->deltaLen to the number of entries there if it's a complete // match. // Set A_End and T_End to the rightmost positions where the // alignment ended in A and T , respectively. // Set Match_To_End true if the match extended to the end // of at least one string; otherwise, set it false to indicate // a branch point. int32 Prefix_Edit_Dist(char *A, int32 m, char *T, int32 n, int32 Error_Limit, int32 &A_End, int32 &T_End, bool &Match_To_End, pedWorkArea_t *WA) { //assert (m <= n); int32 Best_d = 0; int32 Best_e = 0; int32 Longest = 0; WA->deltaLen = 0; int32 shorter = min(m, n); int32 Row = 0; while ((Row < shorter) && (A[Row] == T[Row])) Row++; if (WA->Edit_Array_Lazy[0] == NULL) Allocate_More_Edit_Space(WA); WA->Edit_Array_Lazy[0][0] = Row; // Exact match? if (Row == shorter) { A_End = Row; T_End = Row; Match_To_End = true; return(0); } int32 Left = 0; int32 Right = 0; double Max_Score = 0.0; int32 Max_Score_Len = 0; int32 Max_Score_Best_d = 0; int32 Max_Score_Best_e = 0; for (int32 e=1; e<=Error_Limit; e++) { if (WA->Edit_Array_Lazy[e] == NULL) Allocate_More_Edit_Space(WA); Left = max(Left - 1, -e); Right = min(Right + 1, e); WA->Edit_Array_Lazy[e-1][Left] = -2; WA->Edit_Array_Lazy[e-1][Left-1] = -2; WA->Edit_Array_Lazy[e-1][Right] = -2; WA->Edit_Array_Lazy[e-1][Right+1] = -2; for (int32 d=Left; d<=Right; d++) { Row = 1 + WA->Edit_Array_Lazy[e-1][d]; Row = max(Row, WA->Edit_Array_Lazy[e-1][d-1]); Row = max(Row, WA->Edit_Array_Lazy[e-1][d+1] + 1); while ((Row < m) && (Row + d < n) && (A[Row] == T[Row + d])) Row++; assert(e < WA->Edit_Array_Max); WA->Edit_Array_Lazy[e][d] = Row; if (Row == m || Row + d == n) { // Force last error to be mismatch rather than insertion if ((Row == m) && (1 + WA->Edit_Array_Lazy[e-1][d+1] == WA->Edit_Array_Lazy[e][d]) && (d < Right)) { d++; WA->Edit_Array_Lazy[e][d] = WA->Edit_Array_Lazy[e][d-1]; } A_End = Row; // One past last align position T_End = Row + d; Match_To_End = true; Compute_Delta(WA, e, d, Row); return(e); } } while (Left <= Right && Left < 0 && WA->Edit_Array_Lazy[e][Left] < WA->G->Edit_Match_Limit[e]) Left++; if (Left >= 0) while (Left <= Right && WA->Edit_Array_Lazy[e][Left] + Left < WA->G->Edit_Match_Limit[e]) Left++; if (Left > Right) break; while (Right > 0 && WA->Edit_Array_Lazy[e][Right] + Right < WA->G->Edit_Match_Limit[e]) Right--; if (Right <= 0) while (WA->Edit_Array_Lazy[e][Right] < WA->G->Edit_Match_Limit[e]) Right--; assert (Left <= Right); for (int32 d=Left; d <= Right; d++) if (WA->Edit_Array_Lazy[e][d] > Longest) { Best_d = d; Best_e = e; Longest = WA->Edit_Array_Lazy[e][d]; } int32 Score = Longest * BRANCH_PT_MATCH_VALUE - e; // Assumes BRANCH_PT_MATCH_VALUE - BRANCH_PT_ERROR_VALUE == 1.0 // CorrectOverlaps didn't have the second clause. // Neither did overlapper. if ((Score > Max_Score) && (Best_e <= WA->G->Error_Bound[min(Longest, Longest + Best_d)])) { Max_Score = Score; Max_Score_Len = Longest; Max_Score_Best_d = Best_d; Max_Score_Best_e = Best_e; } } // CorrectOverlaps doesn't have this call Compute_Delta(WA, Max_Score_Best_e, Max_Score_Best_d, Max_Score_Len); A_End = Max_Score_Len; T_End = Max_Score_Len + Max_Score_Best_d; Match_To_End = false; // CorrectOverlaps was returning just 'e', not the best as below. // It used this return value to compute the new error rate. return(Max_Score_Best_e); } canu-1.3/src/overlapErrorAdjustment/findErrors-Process_Olap.C000066400000000000000000000121331272335422300244260ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "findErrors.H" #include "AS_UTL_reverseComplement.H" int32 Prefix_Edit_Dist(char *A, int m, char *T, int n, int Error_Limit, int32 &A_End, int32 &T_End, bool &Match_To_End, pedWorkArea_t * wa); void Analyze_Alignment(Thread_Work_Area_t *wa, char *a_part, int32 a_len, int32 a_offset, char *b_part, int32 b_len, int32 sub); // Find the alignment referred to in olap , where the a_iid // fragment is in Frag and the b_iid sequence is in b_seq . // Use the alignment to increment the appropriate vote fields // for the a fragment. shredded is true iff the b fragment // is from shredded data, in which case the overlap will be // ignored if the a fragment is also shredded. // rev_seq is a buffer to hold the reverse complement of b_seq // if needed. (* rev_id) is used to keep track of whether // rev_seq has been created yet. (* wa) is the work-area // containing space for the process to use in case of multi-threading. void Process_Olap(Olap_Info_t *olap, char *b_seq, bool shredded, Thread_Work_Area_t *wa) { #if 0 fprintf(stderr, "Process_Olap: %8d %8d %5d %5d %c\n", olap->a_iid, olap->b_iid, olap->a_hang, olap->b_hang, olap->innie == true ? 'I' : 'N'); #endif int32 ri = olap->a_iid - wa->G->bgnID; if ((shredded == true) && (wa->G->reads[ri].shredded == true)) return; char *a_part = wa->G->reads[ri].sequence; int32 a_offset = 0; char *b_part = (olap->normal == true) ? b_seq : wa->rev_seq; int32 b_offset = 0; // If innie, reverse-complement the B sequence. if ((olap->innie == true) && (wa->rev_id != olap->b_iid)) { strcpy(b_part, b_seq); reverseComplementSequence(b_part, 0); wa->rev_id = olap->b_iid; } // Adjust for hangs. if (olap->a_hang > 0) { a_offset = olap->a_hang; a_part += a_offset; } if (olap->a_hang < 0) { b_offset = -olap->a_hang; b_part += b_offset; } // Count degree - just how many times we cover the end of the read? if ((olap->a_hang <= 0) && (wa->G->reads[ri].left_degree < MAX_DEGREE)) wa->G->reads[ri].left_degree++; if ((olap->b_hang >= 0) && (wa->G->reads[ri].right_degree < MAX_DEGREE)) wa->G->reads[ri].right_degree++; // Get the alignment uint32 a_part_len = strlen(a_part); uint32 b_part_len = strlen(b_part); int32 a_end = 0; int32 b_end = 0; uint32 olap_len = min(a_part_len, b_part_len); bool match_to_end = false; //fprintf(stderr, "A: offset %d length %d\n", a_offset, a_part_len); //fprintf(stderr, "B: offset %d length %d\n", b_offset, b_part_len); int32 errors = Prefix_Edit_Dist(a_part, a_part_len, b_part, b_part_len, wa->G->Error_Bound[olap_len], a_end, b_end, match_to_end, &wa->ped); if ((a_end < 0) || (a_end > a_part_len) || (b_end < 0) || (b_end > b_part_len)) { fprintf (stderr, "ERROR: Bad edit distance.\n"); fprintf (stderr, " errors = %d a_end = %d b_end = %d\n", errors, a_end, b_end); fprintf (stderr, " a_part_len = %d b_part_len = %d\n", a_part_len, b_part_len); fprintf (stderr, " a_iid = %d b_iid = %d match_to_end = %c\n", olap->a_iid, olap->b_iid, match_to_end ? 'T' : 'F'); } assert(a_end >= 0); assert(a_end <= a_part_len); assert(b_end >= 0); assert(b_end <= b_part_len); //printf(" errors = %d delta_len = %d\n", errors, wa->ped.deltaLen); //printf(" a_align = %d/%d b_align = %d/%d\n", a_end, a_part_len, b_end, b_part_len); //Display_Alignment(a_part, a_end, b_part, b_end, wa->delta, wa->deltaLen, wa->G->reads[ri].clear_len - a_offset); if ((match_to_end == false) && (a_end + a_offset >= wa->G->reads[ri].clear_len - 1)) { olap_len = min(a_end, b_end); match_to_end = true; } if ((errors <= wa->G->Error_Bound[olap_len]) && (match_to_end == true)) Analyze_Alignment(wa, a_part, a_end, a_offset, b_part, b_end, ri); else wa->failedOlaps++; } canu-1.3/src/overlapErrorAdjustment/findErrors-Read_Frags.C000066400000000000000000000077511272335422300240440ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAY-05 to 2015-JUN-03 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "findErrors.H" // Open and read fragments with IIDs from Lo_Frag_IID to // Hi_Frag_IID (INCLUSIVE) from gkpStore_Path and store them in // global Frag . // This shares lots of code with Extract_Needed_Frags. void Read_Frags(feParameters *G, gkStore *gkpStore) { // The original converted to lowercase, and made non-acgt be 'a'. char filter[256]; for (uint32 i=0; i<256; i++) filter[i] = 'a'; filter['A'] = filter['a'] = 'a'; filter['C'] = filter['c'] = 'c'; filter['G'] = filter['g'] = 'g'; filter['T'] = filter['t'] = 't'; // Count the number of bases, so we can do two gigantic allocations for // bases and votes. uint64 basesLength = 0; uint64 votesLength = 0; uint64 readsLoaded = 0; fprintf(stderr, "Read_Frags()-- from "F_U32" through "F_U32"\n", G->bgnID, G->endID); for (uint32 curID=G->bgnID; curID<=G->endID; curID++) { gkRead *read = gkpStore->gkStore_getRead(curID); basesLength += read->gkRead_sequenceLength() + 1; votesLength += read->gkRead_sequenceLength(); } uint64 totAlloc = (sizeof(char) * basesLength + sizeof(Vote_Tally_t) * basesLength + sizeof(Frag_Info_t) * G->readsLen); fprintf(stderr, "Read_Frags()-- allocate %lu MB for bases, votes and info, for %u reads of total length %lu (%.4f bytes/base)\n", totAlloc >> 20, G->endID - G->bgnID + 1, basesLength, (double)totAlloc / basesLength); G->readBases = new char [basesLength]; G->readVotes = new Vote_Tally_t [votesLength]; // NO constructor, MUST INIT G->readsLen = G->endID - G->bgnID + 1; G->reads = new Frag_Info_t [G->readsLen]; // Has constructor, no need to init memset(G->readBases, 0, sizeof(char) * basesLength); memset(G->readVotes, 0, sizeof(Vote_Tally_t) * votesLength); basesLength = 0; votesLength = 0; gkReadData *readData = new gkReadData; for (uint32 curID=G->bgnID; curID<=G->endID; curID++) { gkRead *read = gkpStore->gkStore_getRead(curID); gkpStore->gkStore_loadReadData(read, readData); uint32 readLength = read->gkRead_sequenceLength(); char *readBases = readData->gkReadData_getSequence(); G->reads[curID - G->bgnID].sequence = G->readBases + basesLength; G->reads[curID - G->bgnID].vote = G->readVotes + votesLength; basesLength += readLength + 1; votesLength += readLength; readsLoaded += 1; for (uint32 bb=0; bbreads[curID - G->bgnID].sequence[bb] = filter[readBases[bb]]; G->reads[curID - G->bgnID].sequence[readLength] = 0; // All good reads end. G->reads[curID - G->bgnID].clear_len = readLength; G->reads[curID - G->bgnID].shredded = false; G->reads[curID - G->bgnID].left_degree = 0; G->reads[curID - G->bgnID].right_degree = 0; } delete readData; fprintf(stderr, "Read_Frags()-- from "F_U32" through "F_U32" -- loaded "F_U64" bases in "F_U64" reads.\n", G->bgnID, G->endID-1, basesLength, readsLoaded); } canu-1.3/src/overlapErrorAdjustment/findErrors-Read_Olaps.C000066400000000000000000000041701272335422300240500ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-JUN-16 to 2015-JUN-25 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "findErrors.H" // Load overlaps with aIID from G->bgnID to G->endID. // Overlaps can be unsorted. void Read_Olaps(feParameters *G, gkStore *gkpStore) { ovStore *ovs = new ovStore(G->ovlStorePath, gkpStore); ovs->setRange(G->bgnID, G->endID); uint64 numolaps = ovs->numOverlapsInRange(); fprintf(stderr, "Read_Olaps()-- loading "F_U64" overlaps.\n", numolaps); G->olaps = new Olap_Info_t [numolaps]; G->olapsLen = 0; ovOverlap olap(gkpStore); while (ovs->readOverlap(&olap)) { G->olaps[G->olapsLen].a_iid = olap.a_iid; G->olaps[G->olapsLen].b_iid = olap.b_iid; G->olaps[G->olapsLen].a_hang = olap.a_hang(); G->olaps[G->olapsLen].b_hang = olap.b_hang(); G->olaps[G->olapsLen].innie = (olap.flipped() == true); G->olaps[G->olapsLen].normal = (olap.flipped() == false); // These are violated if the innie/normal members are signed! assert(G->olaps[G->olapsLen].innie != G->olaps[G->olapsLen].normal); assert((G->olaps[G->olapsLen].innie == false) || (G->olaps[G->olapsLen].innie == true)); assert((G->olaps[G->olapsLen].normal == false) || (G->olaps[G->olapsLen].normal == true)); G->olapsLen++; } delete ovs; } canu-1.3/src/overlapErrorAdjustment/findErrors.C000066400000000000000000000342141272335422300220430ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAY-29 to 2015-JUL-01 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "findErrors.H" #include "Binomial_Bound.H" void Process_Olap(Olap_Info_t *olap, char *b_seq, bool shredded, Thread_Work_Area_t *wa); void Read_Frags(feParameters *G, gkStore *gkpStore); void Read_Olaps(feParameters *G, gkStore *gkpStore); void Output_Corrections(feParameters *G); // From overlapInCore.C int Binomial_Bound (int e, double p, int Start, double Limit); // Read fragments lo_frag..hi_frag (INCLUSIVE) from store and save the ids and sequences of those // with overlaps to fragments in global Frag . static void Extract_Needed_Frags(feParameters *G, gkStore *gkpStore, uint32 loID, uint32 hiID, Frag_List_t *fl, uint64 &nextOlap) { // The original converted to lowercase, and made non-acgt be 'a'. char filter[256]; for (uint32 i=0; i<256; i++) filter[i] = 'a'; filter['A'] = filter['a'] = 'a'; filter['C'] = filter['c'] = 'c'; filter['G'] = filter['g'] = 'g'; filter['T'] = filter['t'] = 't'; // Count the amount of stuff we're loading. fl->readsLen = 0; fl->basesLen = 0; uint64 lastOlap = nextOlap; uint32 ii = 0; // Index into reads arrays uint32 fi = G->olaps[lastOlap].b_iid; // Actual ID we're extracting assert(loID <= fi); fprintf(stderr, "Extract_Needed_Frags()-- Loading used reads between "F_U32" and "F_U32".\n", fi, hiID); fprintf(stderr, "Extract_Needed_Frags()-- At overlap "F_U64"\n", lastOlap); while (fi <= hiID) { gkRead *read = gkpStore->gkStore_getRead(fi); fl->readsLen += 1; fl->basesLen += read->gkRead_sequenceLength() + 1; // Advance to the next overlap lastOlap++; while ((lastOlap < G->olapsLen) && (G->olaps[lastOlap].b_iid == fi)) lastOlap++; fi = (lastOlap < G->olapsLen) ? G->olaps[lastOlap].b_iid : hiID + 1; } fprintf(stderr, "Extract_Needed_Frags()-- Loading reads for overlaps "F_U64" to "F_U64"\n", nextOlap, lastOlap); fprintf(stderr, "Extract_Needed_Frags()-- reads "F_U32" bases "F_U64"\n", fl->readsLen, fl->basesLen); // Ensure there is space. if (fl->readsMax < fl->readsLen) { delete [] fl->readIDs; delete [] fl->readBases; fprintf(stderr, "realloc reads from "F_U32" to "F_U32"\n", fl->readsMax, 12 * fl->readsLen / 10); fl->readIDs = new uint32 [12 * fl->readsLen / 10]; fl->readBases = new char * [12 * fl->readsLen / 10]; fl->readsMax = 12 * fl->readsLen / 10; } if (fl->basesMax < fl->basesLen) { delete [] fl->bases; fprintf(stderr, "realloc bases from "F_U64" to "F_U64"\n", fl->basesMax, 12 * fl->basesLen / 10); fl->bases = new char [12 * fl->basesLen / 10]; fl->basesMax = 12 * fl->basesLen / 10; } // Load. This is complicated by loading only the reads that have overlaps we care about. fl->readsLen = 0; fl->basesLen = 0; gkReadData *readData = new gkReadData; ii = 0; fi = G->olaps[nextOlap].b_iid; assert(loID <= fi); while (fi <= hiID) { gkRead *read = gkpStore->gkStore_getRead(fi); fl->readIDs[ii] = fi; fl->readBases[ii] = fl->bases + fl->basesLen; fl->basesLen += read->gkRead_sequenceLength() + 1; gkpStore->gkStore_loadReadData(read, readData); uint32 readLen = read->gkRead_sequenceLength(); char *readBases = readData->gkReadData_getSequence(); for (uint32 bb=0; bbreadBases[ii][bb] = filter[readBases[bb]]; fl->readBases[ii][readLen] = 0; // All good reads end. ii++; // Advance to the next overlap. nextOlap++; while ((nextOlap < G->olapsLen) && (G->olaps[nextOlap].b_iid == fi)) nextOlap++; fi = (nextOlap < G->olapsLen) ? G->olaps[nextOlap].b_iid : hiID + 1; } delete readData; fl->readsLen = ii; fprintf(stderr, "Extract_Needed_Frags()-- Loaded "F_U32" reads (%.4f%%). Loaded IDs "F_U32" through "F_U32".\n", fl->readsLen, 100.0 * fl->readsLen / (hiID - 1 - loID), fl->readIDs[0], fl->readIDs[fl->readsLen-1]); } // Process all old fragments in Internal_gkpStore. Only // do overlaps/corrections with fragments where // frag_iid % Num_PThreads == thread_id void * Threaded_Process_Stream(void *ptr) { Thread_Work_Area_t *wa = (Thread_Work_Area_t *)ptr; for (int32 i=0; ifrag_list->readsLen; i++) { int32 skip_id = -1; while (wa->frag_list->readIDs[i] > wa->G->olaps[wa->nextOlap].b_iid) { if (wa->G->olaps[wa->nextOlap].b_iid != skip_id) { fprintf(stderr, "SKIP: b_iid = %d\n", wa->G->olaps[wa->nextOlap].b_iid); skip_id = wa->G->olaps[wa->nextOlap].b_iid; } wa->nextOlap++; } if (wa->frag_list->readIDs[i] != wa->G->olaps[wa->nextOlap].b_iid) { fprintf (stderr, "ERROR: Lists don't match\n"); fprintf (stderr, "frag_list iid = %d nextOlap = %d i = %d\n", wa->frag_list->readIDs[i], wa->G->olaps[wa->nextOlap].b_iid, i); exit (1); } wa->rev_id = UINT32_MAX; while ((wa->nextOlap < wa->G->olapsLen) && (wa->G->olaps[wa->nextOlap].b_iid == wa->frag_list->readIDs[i])) { if (wa->G->olaps[wa->nextOlap].a_iid % wa->G->numThreads == wa->thread_id) { Process_Olap(wa->G->olaps + wa->nextOlap, wa->frag_list->readBases[i], false, // shredded wa); } wa->nextOlap++; } } //pthread_mutex_lock(& Print_Mutex); //fprintf(stderr, "Thread %d processed %d olaps\n", wa->thread_id, olap_ct); //pthread_mutex_unlock(& Print_Mutex); pthread_exit(ptr); return(NULL); } // Read old fragments in gkpStore that have overlaps with // fragments in Frag. Read a batch at a time and process them // with multiple pthreads. Each thread processes all the old fragments // but only changes entries in Frag that correspond to its thread // ID. Recomputes the overlaps and records the vote information about // changes to make (or not) to fragments in Frag . static void Threaded_Stream_Old_Frags(feParameters *G, gkStore *gkpStore) { pthread_attr_t attr; pthread_mutex_init(&G->Print_Mutex, NULL); pthread_attr_init(&attr); pthread_attr_setstacksize(&attr, THREAD_STACKSIZE); pthread_t *thread_id = new pthread_t [G->numThreads]; Thread_Work_Area_t *thread_wa = new Thread_Work_Area_t[G->numThreads]; for (uint32 i=0; inumThreads; i++) { thread_wa[i].thread_id = i; thread_wa[i].loID = 0; thread_wa[i].hiID = 0; thread_wa[i].nextOlap = 0; thread_wa[i].G = G; thread_wa[i].frag_list = NULL; thread_wa[i].rev_id = UINT32_MAX; thread_wa[i].failedOlaps = 0; memset(thread_wa[i].rev_seq, 0, sizeof(char) * AS_MAX_READLEN); double MAX_ERRORS = 1 + (uint32)(G->errorRate * AS_MAX_READLEN); thread_wa[i].ped.initialize(G, G->errorRate); } uint32 loID = G->olaps[0].b_iid; uint32 hiID = loID + FRAGS_PER_BATCH - 1; uint32 endID = G->olaps[G->olapsLen - 1].b_iid; if (hiID > endID) hiID = endID; uint64 frstOlap = 0; uint64 nextOlap = 0; Frag_List_t frag_list_1; Frag_List_t frag_list_2; Frag_List_t *curr_frag_list = &frag_list_1; Frag_List_t *next_frag_list = &frag_list_2; Extract_Needed_Frags(G, gkpStore, loID, hiID, curr_frag_list, nextOlap); while (loID <= endID) { // Process fragments in curr_frag_list in background for (uint32 i=0; inumThreads; i++) { thread_wa[i].loID = loID; thread_wa[i].hiID = hiID; thread_wa[i].nextOlap = frstOlap; thread_wa[i].frag_list = curr_frag_list; int status = pthread_create(thread_id + i, &attr, Threaded_Process_Stream, thread_wa + i); if (status != 0) fprintf(stderr, "pthread_create error: %s\n", strerror(status)), exit(1); } // Read next batch of fragments loID = hiID + 1; if (loID <= endID) { hiID = loID + FRAGS_PER_BATCH - 1; if (hiID > endID) hiID = endID; frstOlap = nextOlap; Extract_Needed_Frags(G, gkpStore, loID, hiID, next_frag_list, nextOlap); } // Wait for background processing to finish for (uint32 i=0; inumThreads; i++) { void *ptr; int status = pthread_join(thread_id[i], &ptr); if (status != 0) fprintf(stderr, "pthread_join error: %s\n", strerror(status)), exit(1); } // Swap the lists and compute another block { Frag_List_t *s = curr_frag_list; curr_frag_list = next_frag_list; next_frag_list = s; } } } int main(int argc, char **argv) { feParameters *G = new feParameters(); argc = AS_configure(argc, argv); int arg = 1; int err = 0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { G->gkpStorePath = argv[++arg]; } else if (strcmp(argv[arg], "-R") == 0) { G->bgnID = atoi(argv[++arg]); G->endID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-O") == 0) { G->ovlStorePath = argv[++arg]; } else if (strcmp(argv[arg], "-e") == 0) { G->errorRate = atof(argv[++arg]); } else if (strcmp(argv[arg], "-l") == 0) { G->minOverlap = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-o") == 0) { // For 'corrections' file output G->outputFileName = argv[++arg]; } else if (strcmp(argv[arg], "-t") == 0) { G->numThreads = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-d") == 0) { G->Degree_Threshold = strtol(argv[++arg], NULL, 10); } else if (strcmp(argv[arg], "-k") == 0) { G->Kmer_Len = strtol(argv[++arg], NULL, 10); } else if (strcmp(argv[arg], "-p") == 0) { G->Use_Haplo_Ct = FALSE; } else if (strcmp(argv[arg], "-V") == 0) { G->Vote_Qualify_Len = strtol(argv[++arg], NULL, 10); } else if (strcmp(argv[arg], "-x") == 0) { G->End_Exclude_Len = strtol(argv[++arg], NULL, 10); } else { fprintf(stderr, "Unknown option '%s'\n", argv[arg]); err++; } arg++; } if (G->gkpStorePath == NULL) err++; if (G->ovlStorePath == NULL) err++; if (G->numThreads == 0) err++; if (err > 0) { fprintf(stderr, "usage: %s[-ehp][-d DegrThresh][-k KmerLen][-x ExcludeLen]\n", argv[0]); fprintf(stderr, " [-F OlapFile][-S OlapStore][-o CorrectFile]\n"); fprintf(stderr, " [-t NumPThreads][-v VerboseLevel]\n"); fprintf(stderr, " [-V Vote_Qualify_Len]\n"); fprintf(stderr, " \n"); fprintf(stderr, "\n"); fprintf(stderr, "Makes corrections to fragment sequence based on overlaps\n"); fprintf(stderr, "and recomputes overlaps on corrected fragments\n"); fprintf(stderr, "Fragments come from and specify\n"); fprintf(stderr, "the range of fragments to modify\n"); fprintf(stderr, "\n"); fprintf(stderr, "Options:\n"); fprintf(stderr, "-d set keep flag on end of frags with less than this many olaps\n"); fprintf(stderr, "-F specify file of sorted overlaps to use (in the format produced\n"); fprintf(stderr, " by get-olaps\n"); fprintf(stderr, "-h print this message\n"); fprintf(stderr, "-k minimum exact-match region to prevent change\n"); fprintf(stderr, "-o specify output file to hold correction info\n"); fprintf(stderr, "-p don't use haplotype counts to correct\n"); fprintf(stderr, "-S specify the binary overlap store containing overlaps to use\n"); fprintf(stderr, "-t set number of p-threads to use\n"); fprintf(stderr, "-v specify level of verbose outputs, higher is more\n"); fprintf(stderr, "-V specify number of exact match bases around an error to vote to change\n"); fprintf(stderr, "-x length of end of exact match to exclude in preventing change\n"); if (G->gkpStorePath == NULL) fprintf(stderr, "ERROR: no gatekeeper store (-G) supplied.\n"); if (G->ovlStorePath == NULL) fprintf(stderr, "ERROR: no overlap store (-O) supplied.\n"); if (G->numThreads == 0) fprintf(stderr, "ERROR: number of compute threads (-t) must be larger than zero.\n"); exit(1); } // // Initialize Globals // double MAX_ERRORS = 1 + (uint32)(G->errorRate * AS_MAX_READLEN); Initialize_Match_Limit(G->Edit_Match_Limit, G->errorRate, MAX_ERRORS); for (uint32 i = 0; i <= AS_MAX_READLEN; i++) G->Error_Bound[i] = (int)ceil(i * G->errorRate); // // // gkStore *gkpStore = gkStore::gkStore_open(G->gkpStorePath); if (G->bgnID < 1) G->bgnID = 1; if (gkpStore->gkStore_getNumReads() < G->endID) G->endID = gkpStore->gkStore_getNumReads(); Read_Frags(G, gkpStore); Read_Olaps(G, gkpStore); // Now sort them! sort(G->olaps, G->olaps + G->olapsLen); //fprintf (stderr, "Before Stream_Old_Frags Num_Olaps = "F_S64"\n", Num_Olaps); Threaded_Stream_Old_Frags(G, gkpStore); //fprintf (stderr, " Failed overlaps = %d\n", Failed_Olaps); gkpStore->gkStore_close(); //Output_Details(G); Output_Corrections(G); delete G; exit(0); } canu-1.3/src/overlapErrorAdjustment/findErrors.H000066400000000000000000000214211272335422300220440ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include #include "gkStore.H" #include "ovStore.H" #include "correctionOutput.H" #include using namespace std; // Private stuff #define Sign(a) ( ((a) > 0) - ((a) < 0) ) // Value to add for a match in finding branch points // 1.20 was the calculated value for 6% vs 35% error discrimination // Converting to integers didn't make it faster #define BRANCH_PT_MATCH_VALUE 0.272 // Value to add for a mismatch in finding branch points // -2.19 was the calculated value for 6% vs 35% error discrimination // Converting to integers didn't make it faster #define BRANCH_PT_ERROR_VALUE -0.728 // Number of bits to store integer versions of error rates #define ERATE_BITS 16 // The number of errors that are ignored in setting probability // bound for terminating alignment extensions in edit distance // calculations #define ERRORS_FOR_FREE 1 // Factor by which to grow memory in olap array when reading it #define EXPANSION_FACTOR 1.4 // Number of old fragments to read into memory-based fragment // store at a time for processing #define FRAGS_PER_BATCH 100000 // Longest name allowed for a file in the overlap store #define MAX_FILENAME_LEN 1000 // Most errors in any edit distance computation // KNOWN ONLY AT RUN TIME //#define MAX_ERRORS (1 + (int) (AS_OVL_ERROR_RATE * AS_MAX_READLEN)) // Highest number of votes before overflow #define MAX_DEGREE 32767 // Highest number of votes before overflow #define MAX_VOTE 255 // Branch points must be at least this many bases from the // end of the fragment to be reported #define MIN_BRANCH_END_DIST 20 // Branch point tails must fall off from the max by at least // this rate #define MIN_BRANCH_TAIL_SLOPE 0.20 // This many or more votes at the same base indicate // a separate haplotype #define MIN_HAPLO_OCCURS 3 // The amount of memory to allocate for the stack of each thread #define THREAD_STACKSIZE (128 * 512 * 512) struct Vote_Tally_t { uint32 confirmed : 8; uint32 deletes : 8; uint32 a_subst : 8; uint32 c_subst : 8; uint32 g_subst : 8; uint32 t_subst : 8; uint32 no_insert : 8; uint32 a_insert : 8; uint32 c_insert : 8; uint32 g_insert : 8; uint32 t_insert : 8; }; struct Vote_t { int32 frag_sub; int32 align_sub; Vote_Value_t vote_val; }; class Frag_Info_t { public: Frag_Info_t() { sequence = NULL; vote = NULL; clear_len = 0; left_degree = 0; right_degree = 0; shredded = false; unused = false; }; ~Frag_Info_t() { }; char *sequence; Vote_Tally_t *vote; uint64 clear_len : 31; uint64 left_degree : 31; uint64 right_degree : 31; uint64 shredded : 1; // True if shredded read uint64 unused : 1; }; class Olap_Info_t { public: Olap_Info_t() {}; ~Olap_Info_t() {}; uint32 a_iid; uint32 b_iid; int64 a_hang : 31; int64 b_hang : 31; uint64 innie : 1; // was 'orient' with choice INNIE=0 or NORMAL=1 uint64 normal : 1; // so 'normal' always != 'innie' // Sort by increasing b_iid, then increasing a_iid. bool operator<(Olap_Info_t const &that) const { if (b_iid < that.b_iid) return(true); if (b_iid > that.b_iid) return(false); if (a_iid < that.a_iid) return(true); if (a_iid > that.a_iid) return(false); // It is possible, but unlikely, to have two overlaps to the same pair of reads, // if we overlap a5'-b3' and a3'-b5'. I think. return(innie != that.innie); }; }; class Frag_List_t { public: Frag_List_t() { readsMax = 0; readsLen = 0; readIDs = NULL; readBases = NULL; basesMax = 0; basesLen = 0; bases = NULL; }; ~Frag_List_t() { delete [] readIDs; delete [] readBases; delete [] bases; }; uint32 readsMax; uint32 readsLen; uint32 *readIDs; char **readBases; uint64 basesMax; uint64 basesLen; char *bases; // Read sequences, 0 terminated }; class feParameters; class pedWorkArea_t { public: pedWorkArea_t() { G = NULL; memset(delta, 0, sizeof(int32) * AS_MAX_READLEN); memset(deltaStack, 0, sizeof(int32) * AS_MAX_READLEN); deltaLen = 0; Edit_Array_Lazy = NULL; }; ~pedWorkArea_t() { for (uint32 xx=0; xx < alloc.size(); xx++) delete [] alloc[xx]; delete [] Edit_Array_Lazy; }; void initialize(feParameters *G_, double errorRate) { G = G_; Edit_Array_Max = 1 + (uint32)(errorRate * AS_MAX_READLEN); Edit_Array_Lazy = new int32 * [Edit_Array_Max]; memset(Edit_Array_Lazy, 0, sizeof(int32 *) * Edit_Array_Max); }; public: feParameters *G; int32 delta[AS_MAX_READLEN]; // Only need ERATE * READLEN int32 deltaStack[AS_MAX_READLEN]; int32 deltaLen; vector alloc; // Allocated blocks, don't use directly. int32 **Edit_Array_Lazy; // Doled out space. int32 Edit_Array_Max; // Former MAX_ERRORS }; struct Thread_Work_Area_t { int32 thread_id; uint32 loID; uint32 hiID; uint64 nextOlap; feParameters *G; Frag_List_t *frag_list; char rev_seq[AS_MAX_READLEN + 1]; // Used in Process_Olap to hold RC of the B read uint32 rev_id; // Ident of the rev_seq read. Vote_t globalvote[AS_MAX_READLEN]; uint32 failedOlaps; pedWorkArea_t ped; }; class feParameters { public: feParameters() { gkpStorePath = NULL; ovlStorePath = NULL; bgnID = 0; endID = UINT32_MAX; readBases = NULL; readVotes = NULL; reads = NULL; readsLen = 0; olaps = NULL; olapsLen = 0; outputFileName = NULL; numThreads = 4; errorRate = 0.06; minOverlap = 0; // Parameters // Output Degree_Threshold = 2; //, DEFAULT_DEGREE_THRESHOLD; Use_Haplo_Ct = TRUE; // Analyze_Alignment End_Exclude_Len = 3; //DEFAULT_END_EXCLUDE_LEN; Kmer_Len = 9; //DEFAULT_KMER_LEN; Vote_Qualify_Len = 9; //DEFAULT_VOTE_QUALIFY_LEN; }; ~feParameters() { delete [] readBases; delete [] readVotes; delete [] reads; delete [] olaps; }; // Paths to stores char *gkpStorePath; char *ovlStorePath; // Range of IDs to process uint32 bgnID; uint32 endID; char *readBases; Vote_Tally_t *readVotes; Frag_Info_t *reads; uint32 readsLen; // Number of fragments being corrected Olap_Info_t *olaps; uint64 olapsLen; // Number of overlaps being used char *outputFileName; uint32 numThreads; double errorRate; uint32 minOverlap; // This array [e] is the minimum value of Edit_Array [e] [d] to be worth pursuing in edit-distance // computations between guides (only MAX_ERRORS needed) int Edit_Match_Limit [AS_MAX_READLEN + 1]; // Set keep flag on end of fragment if number of olaps < this value int Degree_Threshold; // Set false by -h option to ignore haplotype counts when correcting int Use_Haplo_Ct; // Length of ends of exact-match regions not used in preventing sequence correction int End_Exclude_Len; // Length of minimum exact match in overlap to confirm base pairs int Kmer_Len; // Number of bases surrounding a SNP to vote for change int Vote_Qualify_Len; // This array [i] is the maximum number of errors allowed in a match between sequences of length // i , which is i * MAXERROR_RATE . int Error_Bound [AS_MAX_READLEN + 1]; // To make debugging printout come out together pthread_mutex_t Print_Mutex; }; canu-1.3/src/overlapErrorAdjustment/findErrors.mk000066400000000000000000000013561272335422300222710ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := findErrors SOURCES := findErrors.C \ findErrors-Analyze_Alignment.C \ findErrors-Output.C \ findErrors-Prefix_Edit_Distance.C \ findErrors-Process_Olap.C \ findErrors-Read_Frags.C \ findErrors-Read_Olaps.C SRC_INCDIRS := .. ../AS_UTL ../stores ../overlapInCore/liboverlap TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/overlapInCore-analysis/000077500000000000000000000000001272335422300174665ustar00rootroot00000000000000canu-1.3/src/overlapInCore-analysis/analyze-true-vs-test.pl000066400000000000000000000432441272335422300240550ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Reads true overlaps from the first file, separates the second file into # true/false based on iid. Hangs are NOT checked. my $truePrefix = "BL"; my $testPrefix = "CAordered"; my $trueOverlaps = "$truePrefix/test.ovlStore"; my $testOverlaps = "$testPrefix/test.ovlStore"; my $filtTrue = "filtered.true"; my $filtFalse = "filtered.false"; # # Load read lengths, make sure both assemblies are the same # my @readLengths; my %contained; open(F, "gatekeeper -dumpfragments -tabular $truePrefix/test.gkpStore |"); while () { my @v = split '\s+', $_; $readLengths[$v[1]] = $v[9]; } close(F); open(F, "gatekeeper -dumpfragments -tabular $testPrefix/test.gkpStore |"); while () { my @v = split '\s+', $_; die "IID $v[1] BL $readLengths[$v[1]] != CA $v[9]\n" if ($readLengths[$v[1]] != $v[9]); } close(F); open(F, "< $testPrefix/4-unitigger/best.contains"); while () { my @v = split '\s+', $_; $contained{$v[0]}++; } close(F); # # Load true overlaps # my %true; # If set, id pair is a true overlap my %trueComputed; # If set, id pair is a true overlap, and was computed { my $last; # local storage my @lenIdent; # local storage of length-vs-ident for true overlaps. ident is not known. open(F, "overlapStore -d $trueOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n"; while () { s/^\s+//; s/\s+$//; my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_; if ($last != $aIID) { open(O, "> iid$last.len-vs-ident.dat"); print O @lenIdent; close(O); undef @lenIdent; } my $oLen = computeOverlapLength($aIID, $bIID, $aHang, $bHang); $true{"$aIID-$bIID"} = $oLen; $true{"$bIID-$aIID"} = $oLen; next if (exists($contained{$aIID})); next if (exists($contained{$bIID})); push @lenIdent, "$oLen\t$12.0\n"; $last = $aIID; } close(F); } print STDERR "true: ", scalar(keys %true) / 2, "\n"; # # Process computed overlaps # my %trueH; my %falseH; my %keysH; my @lenIdent5T; my @lenIdent5F; my @lenIdent3T; my @lenIdent3F; my $longestF = 0; my $longestT = 0; { my $last = 0; open(LT, "> all.true.length-vs-ident.dat"); open(LF, "> all.false.length-vs-ident.dat"); open(TO, "> all.true.filtered-overlaps.ova"); open(FO, "> all.false.filtered-overlaps.ova"); open(F, "overlapStore -d $testOverlaps |") or die "Failed to open '$testOverlaps' for reading.\n"; while () { s/^\s+//; s/\s+$//; my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_; if ($last != $aIID) { if (scalar(keys %falseH) > 0) { # error rate histogram if (1) { open(O, "> iid$last.eratehistogram.dat"); foreach my $ii (sort { $a <=> $b } keys %keysH) { $trueH{$ii} += 0.0; $falseH{$ii} += 0.0; print O "$ii\t$trueH{$ii}\t$falseH{$ii}\n"; } close(O); open(O, "| gnuplot > /dev/null 2> /dev/null"); print O "set terminal 'png'\n"; print O "set output 'iid$last.eratehistogram.png'\n"; print O "plot 'iid$last.eratehistogram.dat' using 1:3 with lines title 'FALSE',"; print O " 'iid$last.eratehistogram.dat' using 1:2 with lines title 'TRUE'\n"; close(O); } # length vs identitiy if (1) { #0.5 * $longestT < $longestF) { #print STDERR "IID $last true: len=$longestT false: len=$longestF\n"; open(O, "> iid$last.len-vs-ident.5.true.dat"); print O @lenIdent5T; close(O); open(O, "> iid$last.len-vs-ident.5.false.dat"); print O @lenIdent5F; close(O); open(O, "> iid$last.len-vs-ident.3.true.dat"); print O @lenIdent3T; close(O); open(O, "> iid$last.len-vs-ident.3.false.dat"); print O @lenIdent3F; close(O); open(O, "| gnuplot > /dev/null 2> /dev/null"); print O "set terminal 'png'\n"; print O "set output 'iid$last.len-vs-ident.png'\n"; print O "plot [10:35] [0:15000]"; print O " 'iid$last.len-vs-ident.5.false.dat' using 2:1 title 'FALSE 5' pt 1 lc 1,"; print O " 'iid$last.len-vs-ident.5.true.dat' using 2:1 title 'TRUE 5' pt 1 lc 2,"; print O " 'iid$last.len-vs-ident.3.false.dat' using 2:1 title 'FALSE 3' pt 2 lc 1,"; print O " 'iid$last.len-vs-ident.3.true.dat' using 2:1 title 'TRUE 3' pt 2 lc 2,"; print O " 'iid$last.len-vs-ident.dat' using 2:1 title 'MISSED' pt 3 lc 3\n"; print O "set output 'iid$last.5.len-vs-ident.png'\n"; print O "plot [10:35] [0:15000]"; print O " 'iid$last.len-vs-ident.5.false.dat' using 2:1 title 'FALSE 5' pt 1 lc 1,"; print O " 'iid$last.len-vs-ident.5.true.dat' using 2:1 title 'TRUE 5' pt 1 lc 2\n"; print O "set output 'iid$last.3.len-vs-ident.png'\n"; print O "plot [10:35] [0:15000]"; print O " 'iid$last.len-vs-ident.3.false.dat' using 2:1 title 'FALSE 3' pt 2 lc 1,"; print O " 'iid$last.len-vs-ident.3.true.dat' using 2:1 title 'TRUE 3' pt 2 lc 2\n"; close(O); } } undef %trueH; undef %falseH; undef %keysH; undef @lenIdent5T; undef @lenIdent5F; undef @lenIdent3T; undef @lenIdent3F; $longestT = 0; $longestF = 0; } if (exists($true{"$aIID-$bIID"})) { print TO "$_\n"; } else { print FO "$_\n"; } $trueComputed{"$aIID-$bIID"}++; next if (exists($contained{$aIID})); next if (exists($contained{$bIID})); my $oLen = computeOverlapLength($aIID, $bIID, $aHang, $bHang); my $intErate = int($origErate); my $is5 = ($aHang < 0); # Computing the number of matches instead of actual length makes every overlap shorter # than truth, and they all get flagged in output # #$oLen -= int($oLen * $origErate / 100); if (exists($true{"$aIID-$bIID"})) { $trueH{$intErate}++; $keysH{$intErate}++; print LT "$oLen\t$origErate\n"; if ($is5) { push @lenIdent5T, "$oLen\t$origErate\n"; } else { push @lenIdent3T, "$oLen\t$origErate\n"; } if ($longestT < $oLen) { $longestT = $oLen; } } else { $falseH{$intErate}++; $keysH{$intErate}++; print LF "$oLen\t$origErate\n"; if ($is5) { push @lenIdent5F, "$oLen\t$origErate\n"; } else { push @lenIdent3F, "$oLen\t$origErate\n"; } if ($longestF < $oLen) { $longestF = $oLen; } } $last = $aIID; } close(F); close(LT); close(LF); close(TO); close(FO); } print "convertOverlap -ovl < all.true.filtered-overlaps.ova > all.true.filtered-overlaps.ovb\n"; system("convertOverlap -ovl < all.true.filtered-overlaps.ova > all.true.filtered-overlaps.ovb"); print "convertOverlap -ovl < all.false.filtered-overlaps.ova > all.false.filtered-overlaps.ovb\n"; system("convertOverlap -ovl < all.false.filtered-overlaps.ova > all.false.filtered-overlaps.ovb"); print "rm -rf all.true.filtered-overlaps.ovlStore all.true.filtered-overlaps+false.ovlStore\n"; system("rm -rf all.true.filtered-overlaps.ovlStore all.true.filtered-overlaps+false.ovlStore"); print "overlapStoreBuild -g $testPrefix/test.gkpStore -o all.true.filtered-overlaps.ovlStore -F 1 all.true.filtered-overlaps.ovb\n"; system("overlapStoreBuild -g $testPrefix/test.gkpStore -o all.true.filtered-overlaps.ovlStore -F 1 all.true.filtered-overlaps.ovb"); print "overlapStoreBuild -g $testPrefix/test.gkpStore -o all.true.filtered-overlaps+false.ovlStore -F 1 all.true.filtered-overlaps.ovb all.false.filtered-overlaps.ovb\n"; system("overlapStoreBuild -g $testPrefix/test.gkpStore -o all.true.filtered-overlaps+false.ovlStore -F 1 all.true.filtered-overlaps.ovb all.false.filtered-overlaps.ovb"); my %bestTrueEdge; my %bestTrueContain; open(F, "< $truePrefix/4-unitigger/best.edges") or die; while () { my @v = split '\s+', $_; $bestTrueEdge{"$v[0]-$v[2]"}++; $bestTrueEdge{"$v[2]-$v[0]"}++; $bestTrueEdge{"$v[0]-$v[4]"}++; $bestTrueEdge{"$v[4]-$v[0]"}++; } close(F); open(F, "< $truePrefix/4-unitigger/best.contains") or die; while () { my @v = split '\s+', $_; $bestTrueContain{"$v[0]-$v[3]"}++; } close(F); my %bestTestEdge; my %bestTestContain; open(F, "< $testPrefix/4-unitigger/best.edges") or die; while () { my @v = split '\s+', $_; $bestTestEdge{"$v[0]-$v[2]"}++; $bestTestEdge{"$v[2]-$v[0]"}++; $bestTestEdge{"$v[0]-$v[4]"}++; $bestTestEdge{"$v[4]-$v[0]"}++; } close(F); open(F, "< $testPrefix/4-unitigger/best.contains") or die; while () { my @v = split '\s+', $_; $bestTestContain{"$v[0]-$v[3]"}++; } close(F); # # Output true overlaps that we missed # open(F, "overlapStore -d $trueOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n"; open(O, "> true-overlaps.missed.other.ova"); open(E, "> true-overlaps.missed.edge.ova"); open(C, "> true-overlaps.missed.contain.ova"); while () { s/^\s+//; s/\s+$//; my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_; next if (exists($trueComputed{"$aIID-$bIID"})); if (exists($bestTrueEdge{"$aIID-$bIID"})) { print E "$_\n"; } elsif (exists($bestTrueContain{"$aIID-$bIID"})) { print C "$_\n"; } else { print O "$_\n"; } } close(O); close(F); # # Classify computed overlaps into true/false for each type (best edge, best contain, the others). # open(F, "overlapStore -d $testOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n"; open(BET, "> computed-overlaps.bestE.true.dat"); open(BEN, "> computed-overlaps.bestE.near.dat"); open(BEF, "> computed-overlaps.bestE.false.dat"); open(BCT, "> computed-overlaps.bestC.true.dat"); open(BCN, "> computed-overlaps.bestC.near.dat"); open(BCF, "> computed-overlaps.bestC.false.dat"); open(OTT, "> computed-overlaps.other.true.dat"); open(OTF, "> computed-overlaps.other.false.dat"); open(BETo, "> computed-overlaps.bestE.true.ova"); open(BENo, "> computed-overlaps.bestE.near.ova"); open(BEFo, "> computed-overlaps.bestE.false.ova"); open(BCTo, "> computed-overlaps.bestC.true.ova"); open(BCNo, "> computed-overlaps.bestC.near.ova"); open(BCFo, "> computed-overlaps.bestC.false.ova"); open(OTTo, "> computed-overlaps.other.true.ova"); open(OTFo, "> computed-overlaps.other.false.ova"); while () { s/^\s+//; s/\s+$//; my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_; my $pair = "$aIID-$bIID"; my $len = computeOverlapLength($aIID, $bIID, $aHang, $bHang); $origErate = int(10 * $origErate) / 10; my $diff = ($aIID < $bIID) ? $bIID - $aIID : $aIID - $bIID; die if ($diff < 0); if (exists($bestTestEdge{$pair})) { if (exists($bestTrueEdge{$pair})) { print BET "$origErate\t$len\n"; print BETo "$_\n"; } elsif (exists($true{$pair})) { print BEN "$origErate\t$len\n"; print BENo "$_\n"; } else { print BEF "$origErate\t$len\n"; print BEFo "$_\n"; } } elsif (exists($bestTestContain{$pair})) { if (exists($bestTrueContain{$pair})) { print BCT "$origErate\t$len\n"; print BCTo "$_\n"; } elsif (exists($true{$pair})) { print BCN "$origErate\t$len\n"; print BCNo "$_\n"; } else { print BCF "$origErate\t$len\n"; print BCFo "$_\n"; } } else { if (exists($true{$pair})) { print OTT "$origErate\t$len\n"; print OTTo "$_\n"; } else { print OTF "$origErate\t$len\n"; print OTFo "$_\n"; } } } close(OTF); close(OTFo); close(OTT); close(OTTo); close(BCF); close(BCFo); close(BCN); close(BCNo); close(BCT); close(BCTo); close(BEF); close(BEFo); close(BEN); close(BENo); close(BET); close(BETo); system("awk '{ print \$1 }' computed-overlaps.bestE.true.dat | sort -n | uniq -c > computed-overlaps.bestE.true.eratehist"); system("awk '{ print \$1 }' computed-overlaps.bestE.near.dat | sort -n | uniq -c > computed-overlaps.bestE.near.eratehist"); system("awk '{ print \$1 }' computed-overlaps.bestE.false.dat | sort -n | uniq -c > computed-overlaps.bestE.false.eratehist"); system("awk '{ print \$1 }' computed-overlaps.bestC.true.dat | sort -n | uniq -c > computed-overlaps.bestC.true.eratehist"); system("awk '{ print \$1 }' computed-overlaps.bestC.near.dat | sort -n | uniq -c > computed-overlaps.bestC.near.eratehist"); system("awk '{ print \$1 }' computed-overlaps.bestC.false.dat | sort -n | uniq -c > computed-overlaps.bestC.false.eratehist"); system("awk '{ print \$1 }' computed-overlaps.other.true.dat | sort -n | uniq -c > computed-overlaps.other.true.eratehist"); system("awk '{ print \$1 }' computed-overlaps.other.false.dat | sort -n | uniq -c > computed-overlaps.other.false.eratehist"); open(F, "> tmp.gp"); print F "set terminal png\n"; print F "\n"; print F "set output 'computed-overlaps.bestE.eratehist.png'\n"; print F "set title 'Computed Overlaps, best edges, error rate histogram'\n"; print F "plot 'computed-overlaps.bestE.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n"; print F " 'computed-overlaps.bestE.near.eratehist' using 2:1 title 'near' ps 1.00 lc 3, \\\n"; print F " 'computed-overlaps.bestE.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n"; print F "\n"; print F "set title 'Computed Overlaps, best contains, error rate histogram'\n"; print F "set output 'computed-overlaps.bestC.eratehist.png'\n"; print F "plot 'computed-overlaps.bestC.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n"; print F " 'computed-overlaps.bestC.near.eratehist' using 2:1 title 'near' ps 1.00 lc 3, \\\n"; print F " 'computed-overlaps.bestC.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n"; print F "\n"; print F "set title 'Computed Overlaps, non-best, error rate histogram'\n"; print F "set output 'computed-overlaps.other.eratehist.png'\n"; print F "plot 'computed-overlaps.other.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n"; print F " 'computed-overlaps.other.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n"; print F "\n"; print F "\n"; print F "set title 'Computed Overlaps, best edges non-best, error-vs-length'\n"; print F "set output 'computed-overlaps.bestE.length-vs-erate.png'\n"; print F "plot 'computed-overlaps.bestE.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n"; print F " 'computed-overlaps.bestE.near.dat' using 1:2 title 'near' ps 0.25 lc 3, \\\n"; print F " 'computed-overlaps.bestE.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n"; print F "\n"; print F "set title 'Computed Overlaps, non-best, error-vs-length'\n"; print F "set output 'computed-overlaps.bestC.length-vs-erate.png'\n"; print F "plot 'computed-overlaps.bestC.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n"; print F " 'computed-overlaps.bestC.near.dat' using 1:2 title 'near' ps 0.25 lc 3, \\\n"; print F " 'computed-overlaps.bestC.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n"; print F "\n"; print F "set title 'Computed Overlaps, non-best, error-vs-length'\n"; print F "set output 'computed-overlaps.other.length-vs-erate.png'\n"; print F "plot 'computed-overlaps.other.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n"; print F " 'computed-overlaps.other.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n"; close(F); system("gnuplot tmp.gp && rm tmp.gp"); sub computeOverlapLength ($$$$) { my ($aIID, $bIID, $aHang, $bHang) = @_; my $aLen = $readLengths[$aIID]; my $bLen = $readLengths[$bIID]; my $aOvl = 0; my $bOvl = 0; # Swiped from bogart if ($aHang < 0) { # bHang < 0 ? ---------- : ---- # ? ---------- : ---------- # $aOvl = ($bHang < 0) ? ($aLen + $bHang) : ($aLen); $bOvl = ($bHang < 0) ? ($bLen + $aHang) : ($bLen + $aHang - $bHang); } else { # bHang < 0 ? ---------- : ---------- # ? ---- : ---------- # $aOvl = ($bHang < 0) ? ($aLen - $aHang + $bHang) : ($aLen - $aHang); $bOvl = ($bHang < 0) ? ($bLen) : ($bLen - $bHang); } return(($aOvl + $bOvl) / 2); } canu-1.3/src/overlapInCore-analysis/check-ordered-reads-for-missed-overlaps.pl000066400000000000000000000063251272335422300275230ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $dataset; $dataset = "BL"; $dataset = "CAordered"; my $maxB = 0; my $maxA = 0; sub processIIDs ($@) { my $thisIID = shift @_; my @iids = sort { $a <=> $b } @_; return if (scalar(@iids) == 0); my $minIID = $iids[0]; my $curIID = 0; my $maxIID = $iids[scalar(@iids) - 1]; my $expectedB = 0; my $expectedA = 0; my $foundB = 0; my $foundA = 0; my $missingB = 0; my $missingA = 0; foreach my $iid (@iids) { #print "$iid -- $curIID\n"; next if ($iid < $thisIID - 30); # Ignore false overlaps next if ($iid > $thisIID + 30); $expectedB = $thisIID - $iid if ($expectedB < $thisIID - $iid); $expectedA = $iid - $thisIID if ($expectedA < $iid - $thisIID); $curIID = $iid - 1 if ($curIID == 0); # If curIID==0, first time here, set to make no missing. if ($iid < $thisIID) { $foundB++; if ($curIID + 1 < $iid) { #print STDERR " missingB $curIID - $iid\n"; $missingB += $iid - 1 - $curIID; } } else { $foundA++; if ($curIID + 1 < $iid) { print STDERR " missingA $curIID - $iid\n"; $missingA += $iid - 1 - $curIID; } } $curIID = $iid; $curIID++ if ($curIID + 1 == $thisIID); } my $found = $foundB + $foundA; my $expected = $expectedB + $expectedA; my $missing = $missingB + $missingA; if ($missingA > 0) { my $fracB = int(10000 * $missingB / $expectedB) / 100; my $fracA = int(10000 * $missingA / $expectedA) / 100; print "$thisIID range $expectedB/$expectedA found $foundB/$foundA missing $missingB/$missingA $fracB%/$fracA%\n"; } } my $lastIID = 0; my @iids; open(F, "overlapStore -d JUNKTEST3/$dataset/test.ovlStore |"); while () { s/^\s+//; s/\s+$//; # v[0] = iid # v[1] = iid # v[2] = orient # v[3] = hang # v[4] = hang # v[5] = ident # v[6] = ident my @v = split '\s+', $_; if ($v[0] != $lastIID) { #print STDERR "PROCESS $lastIID ", scalar(@iids), "\n"; processIIDs($lastIID, @iids); undef @iids; $lastIID = $v[0]; } #print STDERR "$v[0] $v[1]\n"; push @iids, $v[1]; } close(F); canu-1.3/src/overlapInCore-analysis/filterTrue.pl000066400000000000000000000430641272335422300221570ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Reads true overlaps from the first file, separates the second file into # true/false based on iid. Hangs are NOT checked. my $trueOverlaps = "BL/test.ovlStore"; my $testOverlaps = "CAordered/test.ovlStore"; my $filtTrue = "filtered.true"; my $filtFalse = "filtered.false"; # # Load read lengths, make sure both assemblies are the same # my @readLengths; my %contained; open(F, "gatekeeper -dumpfragments -tabular BL/test.gkpStore |"); while () { my @v = split '\s+', $_; $readLengths[$v[1]] = $v[9]; } close(F); open(F, "gatekeeper -dumpfragments -tabular CAordered/test.gkpStore |"); while () { my @v = split '\s+', $_; die "IID $v[1] BL $readLengths[$v[1]] != CA $v[9]\n" if ($readLengths[$v[1]] != $v[9]); } close(F); open(F, "< CAordered/4-unitigger/best.contains"); while () { my @v = split '\s+', $_; $contained{$v[0]}++; } close(F); # # Load true overlaps # my %true; # If set, id pair is a true overlap my %trueComputed; # If set, id pair is a true overlap, and was computed { my $last; # local storage my @lenIdent; # local storage of length-vs-ident for true overlaps. ident is not known. open(F, "overlapStore -d $trueOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n"; while () { s/^\s+//; s/\s+$//; my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_; if ($last != $aIID) { open(O, "> iid$last.len-vs-ident.dat"); print O @lenIdent; close(O); undef @lenIdent; } my $oLen = computeOverlapLength($aIID, $bIID, $aHang, $bHang); $true{"$aIID-$bIID"} = $oLen; $true{"$bIID-$aIID"} = $oLen; next if (exists($contained{$aIID})); next if (exists($contained{$bIID})); push @lenIdent, "$oLen\t$12.0\n"; $last = $aIID; } close(F); } print STDERR "true: ", scalar(keys %true) / 2, "\n"; # # Process computed overlaps # my %trueH; my %falseH; my %keysH; my @lenIdent5T; my @lenIdent5F; my @lenIdent3T; my @lenIdent3F; my $longestF = 0; my $longestT = 0; { my $last = 0; open(LT, "> all.true.length-vs-ident.dat"); open(LF, "> all.false.length-vs-ident.dat"); open(TO, "> all.true.filtered-overlaps.ova"); open(FO, "> all.false.filtered-overlaps.ova"); open(F, "overlapStore -d $testOverlaps |") or die "Failed to open '$testOverlaps' for reading.\n"; while () { s/^\s+//; s/\s+$//; my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_; if ($last != $aIID) { if (scalar(keys %falseH) > 0) { # error rate histogram if (1) { open(O, "> iid$last.eratehistogram.dat"); foreach my $ii (sort { $a <=> $b } keys %keysH) { $trueH{$ii} += 0.0; $falseH{$ii} += 0.0; print O "$ii\t$trueH{$ii}\t$falseH{$ii}\n"; } close(O); open(O, "| gnuplot > /dev/null 2> /dev/null"); print O "set terminal 'png'\n"; print O "set output 'iid$last.eratehistogram.png'\n"; print O "plot 'iid$last.eratehistogram.dat' using 1:3 with lines title 'FALSE',"; print O " 'iid$last.eratehistogram.dat' using 1:2 with lines title 'TRUE'\n"; close(O); } # length vs identitiy if (1) { #0.5 * $longestT < $longestF) { #print STDERR "IID $last true: len=$longestT false: len=$longestF\n"; open(O, "> iid$last.len-vs-ident.5.true.dat"); print O @lenIdent5T; close(O); open(O, "> iid$last.len-vs-ident.5.false.dat"); print O @lenIdent5F; close(O); open(O, "> iid$last.len-vs-ident.3.true.dat"); print O @lenIdent3T; close(O); open(O, "> iid$last.len-vs-ident.3.false.dat"); print O @lenIdent3F; close(O); open(O, "| gnuplot > /dev/null 2> /dev/null"); print O "set terminal 'png'\n"; print O "set output 'iid$last.len-vs-ident.png'\n"; print O "plot [10:35] [0:15000]"; print O " 'iid$last.len-vs-ident.5.false.dat' using 2:1 title 'FALSE 5' pt 1 lc 1,"; print O " 'iid$last.len-vs-ident.5.true.dat' using 2:1 title 'TRUE 5' pt 1 lc 2,"; print O " 'iid$last.len-vs-ident.3.false.dat' using 2:1 title 'FALSE 3' pt 2 lc 1,"; print O " 'iid$last.len-vs-ident.3.true.dat' using 2:1 title 'TRUE 3' pt 2 lc 2,"; print O " 'iid$last.len-vs-ident.dat' using 2:1 title 'MISSED' pt 3 lc 3\n"; print O "set output 'iid$last.5.len-vs-ident.png'\n"; print O "plot [10:35] [0:15000]"; print O " 'iid$last.len-vs-ident.5.false.dat' using 2:1 title 'FALSE 5' pt 1 lc 1,"; print O " 'iid$last.len-vs-ident.5.true.dat' using 2:1 title 'TRUE 5' pt 1 lc 2\n"; print O "set output 'iid$last.3.len-vs-ident.png'\n"; print O "plot [10:35] [0:15000]"; print O " 'iid$last.len-vs-ident.3.false.dat' using 2:1 title 'FALSE 3' pt 2 lc 1,"; print O " 'iid$last.len-vs-ident.3.true.dat' using 2:1 title 'TRUE 3' pt 2 lc 2\n"; close(O); } } undef %trueH; undef %falseH; undef %keysH; undef @lenIdent5T; undef @lenIdent5F; undef @lenIdent3T; undef @lenIdent3F; $longestT = 0; $longestF = 0; } if (exists($true{"$aIID-$bIID"})) { print TO "$_\n"; } else { print FO "$_\n"; } $trueComputed{"$aIID-$bIID"}++; next if (exists($contained{$aIID})); next if (exists($contained{$bIID})); my $oLen = computeOverlapLength($aIID, $bIID, $aHang, $bHang); my $intErate = int($origErate); my $is5 = ($aHang < 0); # Computing the number of matches instead of actual length makes every overlap shorter # than truth, and they all get flagged in output # #$oLen -= int($oLen * $origErate / 100); if (exists($true{"$aIID-$bIID"})) { $trueH{$intErate}++; $keysH{$intErate}++; print LT "$oLen\t$origErate\n"; if ($is5) { push @lenIdent5T, "$oLen\t$origErate\n"; } else { push @lenIdent3T, "$oLen\t$origErate\n"; } if ($longestT < $oLen) { $longestT = $oLen; } } else { $falseH{$intErate}++; $keysH{$intErate}++; print LF "$oLen\t$origErate\n"; if ($is5) { push @lenIdent5F, "$oLen\t$origErate\n"; } else { push @lenIdent3F, "$oLen\t$origErate\n"; } if ($longestF < $oLen) { $longestF = $oLen; } } $last = $aIID; } close(F); close(LT); close(LF); close(TO); close(FO); } print "convertOverlap -ovl < all.true.filtered-overlaps.ova > all.true.filtered-overlaps.ovb\n"; system("convertOverlap -ovl < all.true.filtered-overlaps.ova > all.true.filtered-overlaps.ovb"); print "convertOverlap -ovl < all.false.filtered-overlaps.ova > all.false.filtered-overlaps.ovb\n"; system("convertOverlap -ovl < all.false.filtered-overlaps.ova > all.false.filtered-overlaps.ovb"); print "rm -rf all.true.filtered-overlaps.ovlStore all.true.filtered-overlaps+false.ovlStore\n"; system("rm -rf all.true.filtered-overlaps.ovlStore all.true.filtered-overlaps+false.ovlStore"); print "overlapStoreBuild -g CAordered/test.gkpStore -o all.true.filtered-overlaps.ovlStore -F 1 all.true.filtered-overlaps.ovb\n"; system("overlapStoreBuild -g CAordered/test.gkpStore -o all.true.filtered-overlaps.ovlStore -F 1 all.true.filtered-overlaps.ovb"); print "overlapStoreBuild -g CAordered/test.gkpStore -o all.true.filtered-overlaps+false.ovlStore -F 1 all.true.filtered-overlaps.ovb all.false.filtered-overlaps.ovb\n"; system("overlapStoreBuild -g CAordered/test.gkpStore -o all.true.filtered-overlaps+false.ovlStore -F 1 all.true.filtered-overlaps.ovb all.false.filtered-overlaps.ovb"); my %bestTrueEdge; my %bestTrueContain; open(F, "< BL/4-unitigger/best.edges") or die; while () { my @v = split '\s+', $_; $bestTrueEdge{"$v[0]-$v[2]"}++; $bestTrueEdge{"$v[2]-$v[0]"}++; $bestTrueEdge{"$v[0]-$v[4]"}++; $bestTrueEdge{"$v[4]-$v[0]"}++; } close(F); open(F, "< BL/4-unitigger/best.contains") or die; while () { my @v = split '\s+', $_; $bestTrueContain{"$v[0]-$v[3]"}++; } close(F); my %bestTestEdge; my %bestTestContain; open(F, "< CAordered/4-unitigger/best.edges") or die; while () { my @v = split '\s+', $_; $bestTestEdge{"$v[0]-$v[2]"}++; $bestTestEdge{"$v[2]-$v[0]"}++; $bestTestEdge{"$v[0]-$v[4]"}++; $bestTestEdge{"$v[4]-$v[0]"}++; } close(F); open(F, "< CAordered/4-unitigger/best.contains") or die; while () { my @v = split '\s+', $_; $bestTestContain{"$v[0]-$v[3]"}++; } close(F); # # Output true overlaps that we missed # open(F, "overlapStore -d $trueOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n"; open(O, "> true-overlaps.missed.other.ova"); open(E, "> true-overlaps.missed.edge.ova"); open(C, "> true-overlaps.missed.contain.ova"); while () { s/^\s+//; s/\s+$//; my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_; next if (exists($trueComputed{"$aIID-$bIID"})); if (exists($bestTrueEdge{"$aIID-$bIID"})) { print E "$_\n"; } elsif (exists($bestTrueContain{"$aIID-$bIID"})) { print C "$_\n"; } else { print O "$_\n"; } } close(O); close(F); # # Classify computed overlaps into true/false for each type (best edge, best contain, the others). # open(F, "overlapStore -d $testOverlaps |") or die "Failed to open '$trueOverlaps' for reading.\n"; open(BET, "> computed-overlaps.bestE.true.dat"); open(BEN, "> computed-overlaps.bestE.near.dat"); open(BEF, "> computed-overlaps.bestE.false.dat"); open(BCT, "> computed-overlaps.bestC.true.dat"); open(BCN, "> computed-overlaps.bestC.near.dat"); open(BCF, "> computed-overlaps.bestC.false.dat"); open(OTT, "> computed-overlaps.other.true.dat"); open(OTF, "> computed-overlaps.other.false.dat"); open(BETo, "> computed-overlaps.bestE.true.ova"); open(BENo, "> computed-overlaps.bestE.near.ova"); open(BEFo, "> computed-overlaps.bestE.false.ova"); open(BCTo, "> computed-overlaps.bestC.true.ova"); open(BCNo, "> computed-overlaps.bestC.near.ova"); open(BCFo, "> computed-overlaps.bestC.false.ova"); open(OTTo, "> computed-overlaps.other.true.ova"); open(OTFo, "> computed-overlaps.other.false.ova"); while () { s/^\s+//; s/\s+$//; my ($aIID, $bIID, $orient, $aHang, $bHang, $origErate, $corrErate) = split '\s+', $_; my $pair = "$aIID-$bIID"; my $len = computeOverlapLength($aIID, $bIID, $aHang, $bHang); $origErate = int(10 * $origErate) / 10; my $diff = ($aIID < $bIID) ? $bIID - $aIID : $aIID - $bIID; die if ($diff < 0); if (exists($bestTestEdge{$pair})) { if (exists($bestTrueEdge{$pair})) { print BET "$origErate\t$len\n"; print BETo "$_\n"; } elsif (exists($true{$pair})) { print BEN "$origErate\t$len\n"; print BENo "$_\n"; } else { print BEF "$origErate\t$len\n"; print BEFo "$_\n"; } } elsif (exists($bestTestContain{$pair})) { if (exists($bestTrueContain{$pair})) { print BCT "$origErate\t$len\n"; print BCTo "$_\n"; } elsif (exists($true{$pair})) { print BCN "$origErate\t$len\n"; print BCNo "$_\n"; } else { print BCF "$origErate\t$len\n"; print BCFo "$_\n"; } } else { if (exists($true{$pair})) { print OTT "$origErate\t$len\n"; print OTTo "$_\n"; } else { print OTF "$origErate\t$len\n"; print OTFo "$_\n"; } } } close(OTF); close(OTFo); close(OTT); close(OTTo); close(BCF); close(BCFo); close(BCN); close(BCNo); close(BCT); close(BCTo); close(BEF); close(BEFo); close(BEN); close(BENo); close(BET); close(BETo); system("awk '{ print \$1 }' computed-overlaps.bestE.true.dat | sort -n | uniq -c > computed-overlaps.bestE.true.eratehist"); system("awk '{ print \$1 }' computed-overlaps.bestE.near.dat | sort -n | uniq -c > computed-overlaps.bestE.near.eratehist"); system("awk '{ print \$1 }' computed-overlaps.bestE.false.dat | sort -n | uniq -c > computed-overlaps.bestE.false.eratehist"); system("awk '{ print \$1 }' computed-overlaps.bestC.true.dat | sort -n | uniq -c > computed-overlaps.bestC.true.eratehist"); system("awk '{ print \$1 }' computed-overlaps.bestC.near.dat | sort -n | uniq -c > computed-overlaps.bestC.near.eratehist"); system("awk '{ print \$1 }' computed-overlaps.bestC.false.dat | sort -n | uniq -c > computed-overlaps.bestC.false.eratehist"); system("awk '{ print \$1 }' computed-overlaps.other.true.dat | sort -n | uniq -c > computed-overlaps.other.true.eratehist"); system("awk '{ print \$1 }' computed-overlaps.other.false.dat | sort -n | uniq -c > computed-overlaps.other.false.eratehist"); open(F, "> tmp.gp"); print F "set terminal png\n"; print F "\n"; print F "set output 'computed-overlaps.bestE.eratehist.png'\n"; print F "set title 'Computed Overlaps, best edges, error rate histogram'\n"; print F "plot 'computed-overlaps.bestE.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n"; print F " 'computed-overlaps.bestE.near.eratehist' using 2:1 title 'near' ps 1.00 lc 3, \\\n"; print F " 'computed-overlaps.bestE.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n"; print F "\n"; print F "set title 'Computed Overlaps, best contains, error rate histogram'\n"; print F "set output 'computed-overlaps.bestC.eratehist.png'\n"; print F "plot 'computed-overlaps.bestC.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n"; print F " 'computed-overlaps.bestC.near.eratehist' using 2:1 title 'near' ps 1.00 lc 3, \\\n"; print F " 'computed-overlaps.bestC.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n"; print F "\n"; print F "set title 'Computed Overlaps, non-best, error rate histogram'\n"; print F "set output 'computed-overlaps.other.eratehist.png'\n"; print F "plot 'computed-overlaps.other.true.eratehist' using 2:1 title 'true' ps 1.00 lc 2, \\\n"; print F " 'computed-overlaps.other.false.eratehist' using 2:1 title 'false' ps 1.00 lc 1\n"; print F "\n"; print F "\n"; print F "set title 'Computed Overlaps, best edges non-best, error-vs-length'\n"; print F "set output 'computed-overlaps.bestE.length-vs-erate.png'\n"; print F "plot 'computed-overlaps.bestE.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n"; print F " 'computed-overlaps.bestE.near.dat' using 1:2 title 'near' ps 0.25 lc 3, \\\n"; print F " 'computed-overlaps.bestE.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n"; print F "\n"; print F "set title 'Computed Overlaps, non-best, error-vs-length'\n"; print F "set output 'computed-overlaps.bestC.length-vs-erate.png'\n"; print F "plot 'computed-overlaps.bestC.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n"; print F " 'computed-overlaps.bestC.near.dat' using 1:2 title 'near' ps 0.25 lc 3, \\\n"; print F " 'computed-overlaps.bestC.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n"; print F "\n"; print F "set title 'Computed Overlaps, non-best, error-vs-length'\n"; print F "set output 'computed-overlaps.other.length-vs-erate.png'\n"; print F "plot 'computed-overlaps.other.true.dat' using 1:2 title 'true' ps 0.50 lc 2, \\\n"; print F " 'computed-overlaps.other.false.dat' using 1:2 title 'false' ps 0.25 lc 1\n"; close(F); system("gnuplot tmp.gp && rm tmp.gp"); sub computeOverlapLength ($$$$) { my ($aIID, $bIID, $aHang, $bHang) = @_; my $aLen = $readLengths[$aIID]; my $bLen = $readLengths[$bIID]; my $aOvl = 0; my $bOvl = 0; # Swiped from bogart if ($aHang < 0) { # bHang < 0 ? ---------- : ---- # ? ---------- : ---------- # $aOvl = ($bHang < 0) ? ($aLen + $bHang) : ($aLen); $bOvl = ($bHang < 0) ? ($bLen + $aHang) : ($bLen + $aHang - $bHang); } else { # bHang < 0 ? ---------- : ---------- # ? ---- : ---------- # $aOvl = ($bHang < 0) ? ($aLen - $aHang + $bHang) : ($aLen - $aHang); $bOvl = ($bHang < 0) ? ($bLen) : ($bLen - $bHang); } return(($aOvl + $bOvl) / 2); } canu-1.3/src/overlapInCore-analysis/find-missed-true-overlaps.pl000066400000000000000000000060061272335422300250350ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Given: # an assembly of true inferred overlaps # an assembly of computed overlaps # report the number of true overlaps that are missed in the computed overlaps my %truthOverlaps; my @readLengths; open(F, "gatekeeper -dumpfragments -tabular JUNKTEST3/BL/test.gkpStore |"); while () { my @v = split '\s+', $_; $readLengths[$v[1]] = $v[9]; } close(F); open(F, "gatekeeper -dumpfragments -tabular JUNKTEST3/CAordered/test.gkpStore |"); while () { my @v = split '\s+', $_; die "IID $v[1] BL $readLengths[$v[1]] != CA $v[9]\n" if ($readLengths[$v[1]] != $v[9]); } close(F); open(F, "overlapStore -d JUNKTEST3/BL/test.ovlStore |"); while () { s/^\s+//; s/\s+$//; my ($aIID, $bIID, $orient, $aHang, $bHang, $origID, $corrID) = split '\s+', $_; my $aLen = $readLengths[$aIID]; my $bLen = $readLengths[$bIID]; my $aOvl = 0; my $bOvl = 0; my $oLen = 0; # Swiped from bogart if ($aHang < 0) { # bHang < 0 ? ---------- : ---- # ? ---------- : ---------- # $aOvl = ($bHang < 0) ? ($aLen + $bHang) : ($aLen); $bOvl = ($bHang < 0) ? ($bLen + $aHang) : ($bLen + $aHang - $bHang); } else { # bHang < 0 ? ---------- : ---------- # ? ---- : ---------- # $aOvl = ($bHang < 0) ? ($aLen - $aHang + $bHang) : ($aLen - $aHang); $bOvl = ($bHang < 0) ? ($bLen) : ($bLen - $bHang); } $oLen = ($aOvl + $bOvl) / 2; $truthOverlaps{"$aIID-$bIID"}++; } close(F); print STDERR "truthOverlaps: ", scalar(keys %truthOverlaps), "\n"; open(F, "overlapStore -d JUNKTEST3/CAordered/test.ovlStore |"); while () { s/^\s+//; s/\s+$//; # v[0] = iid # v[1] = iid # v[2] = orient # v[3] = hang # v[4] = hang # v[5] = ident # v[6] = ident my ($aIID, $bIID, $orient, $aHang, $bHang, $origID, $corrID) = split '\s+', $_; delete $truthOverlaps{"$aIID-$bIID"}; } close(F); print STDERR "truthOverlaps missed: ", scalar(keys %truthOverlaps), "\n"; canu-1.3/src/overlapInCore-analysis/infer-obt-from-genomic-blasr.pl000066400000000000000000000034741272335422300254010ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Convert BLASR default output to OBT overlaps: aIID bIID [f|r] aBgn aEnd bBgn bEnd error my %IDmap; open(F, "< $ARGV[0]") or die "Failed to open '$ARGV[0]' for reading sequence names.\n"; while () { my ($uid, $iid, $name) = split '\s+', $_; $IDmap{$name} = $iid; } close(F); while () { my @v = split '\s+', $_; # The first read seems to have a sub-read range appended. if ($v[0] =~ m/^(.*)\/\d+_\d+$/) { $v[0] = $1; } my $aiid = $IDmap{$v[0]}; my $biid = $IDmap{$v[1]}; my $fr = ($v[8] == 0) ? "f" : "r"; my $error = 100.0 - $v[3]; ($v[9], $v[10]) = ($v[10], $v[9]) if ($v[8] == 1); die "No A iid found for '$v[0]'\n" if (!defined($aiid)); die "No B iid found for '$v[1]'\n" if (!defined($biid)); next if ($aiid == $biid); # HACK! #$error = 0.01; print "$aiid\t$biid\t$fr\t$v[5]\t$v[6]\t$v[9]\t$v[10]\t$error\n"; } canu-1.3/src/overlapInCore-analysis/infer-olaps-from-genomic-coords.pl000066400000000000000000000177541272335422300261270ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $outputPrefix = $ARGV[0]; my $readsFile = $ARGV[1]; my $coordsFile = $ARGV[2]; my $uidMapFile = $ARGV[3]; my $orientForward = 0; # # Load name to IID map # my %NAMEtoIID; my %UIDtoNAME; if (defined($uidMapFile)) { print STDERR "Read names from '$uidMapFile'\n"; open(F, "< $uidMapFile"); while () { my @v = split '\s+', $_; # UID IID NAME UID IID NAME if (scalar(@v) == 3) { $NAMEtoIID{$v[2]} = $v[1]; $UIDtoNAME{$v[0]} = $v[2]; } if (scalar(@v) == 6) { $NAMEtoIID{$v[2]} = $v[1]; $NAMEtoIID{$v[5]} = $v[4]; $UIDtoNAME{$v[0]} = $v[2]; $UIDtoNAME{$v[3]} = $v[5]; } } close(F); } # # Load read lengths. Needed to find reads that do not map full length, and to compute hangs. # Assumes reads are dumped from gkpStore and have names 'uid,iid'. my %readLength; my %totalAligns; my %goodAligns; print STDERR "Read lengths from '$readsFile'\n"; open(F, "< $readsFile") or die "Couldn't read '$readsFile'\n"; while (!eof(F)) { my $a = ; my $b = ; my $c = ; my $d = ; if ($a =~ m/^.(\S+)$/) { if (! exists($NAMEtoIID{$1})) { #print STDERR "WARNING: read '$1' not in map.\n" } else { $readLength{$1} = length($b) - 1; $totalAligns{$1} = 0; $goodAligns{$1} = 0; } } else { chomp; die "Failed to parse read name from '$a'\n"; } } close(F); my @aligns; my $totalAligns = 0; my $goodAligns = 0; my $readOrder = 0; my %positions; open(F, "< $coordsFile"); $_ = ; $_ = ; $_ = ; $_ = ; while () { my @v = split '\s+', $_; my $b = $v[0]; # reference begin my $e = $v[1]; # reference end my $r = $v[9]; # reference name my $n = $v[10]; # read name my $f = 1; my $L = $v[2]; # read begin, unused my $R = $v[3]; # read end, unused if ($R < $L) { $f = 0; $L = $v[3]; $R = $v[2]; } die "L=$L R=$R\n" if ($R < $L); die "b=$b e=$e\n" if ($e < $b); next if (!exists($readLength{$n})); die "Didn't find read length for read '$n'\n" if (!exists($readLength{$n})); $totalAligns++; $totalAligns{$n}++; next if (10 < $L); next if (10 < $readLength{$n} - $R); $goodAligns++; $goodAligns{$n}++; my $scale = ($R - $L) / ($e - $b); # scale from reference coords to read coords # The aligns line starts with reference name. This lets us group aligns per reference sequence. push @aligns, "$r\0$b\0$e\0$f\0$scale\0$n"; } # Generate olaps. We sort by reference name, then step through all aligns to that reference and # sort by begin coordinate. my $olaps = 0; open(OVA, "> $outputPrefix.ova"); @aligns = sort @aligns; while (scalar(@aligns) > 0) { my @coords; my @v = split '\0', $aligns[0]; my $r = $v[0]; # Find all coords for this reference sequence while ($r eq $v[0]) { push @coords, "$v[1]\0$v[2]\0$v[3]\0$v[4]\0$v[5]"; shift @aligns; @v = split '\0', $aligns[0]; } # Sort those coords by start position @coords = sort { $a <=> $b } @coords; # Process each one while (scalar(@coords) > 0) { my ($rb, $re, $rf, $rs, $rn) = split '\0', $coords[0]; die "Reversed $coords[0]\n" if ($re < $rb); next if (!exists($NAMEtoIID{$rn})); # Just skip reads we don't care about. die if (!exists($NAMEtoIID{$rn})); # For development, fail. # Save this reads position if (exists($positions{$rn})) { print STDERR "WARNING: read $rn already has a position!\n"; } $positions{$rn} = $readOrder++; # Relabel the read name to an IID. $rn = $NAMEtoIID{$rn}; shift @coords; foreach my $cc (@coords) { my ($cb, $ce, $cf, $cs, $cn) = split '\0', $cc; my $olap = $re - $cb; last if ($olap < 40); my $ahang; my $bhang; if ($rf == 1) { # Read is forward $ahang = $cb - $rb; $bhang = $ce - $re; } else { # Read is reverse; hangs are swapped and negative from forward case $ahang = $re - $ce; $bhang = $rb - $cb; } next if (!exists($NAMEtoIID{$cn})); die if (!exists($NAMEtoIID{$cn})); $cn = $NAMEtoIID{$cn}; # Scale the hangs. Hangs are computed using genomic positions, but deletions from the read # cause this hang to be too large. if ($ahang < 0) { $ahang = $ahang * $cs; # hang is composed of the C read } else { $ahang = $ahang * $rs; # hang is composed of the R read } if ($bhang < 0) { $bhang = $bhang * $rs; # hang is composed of the R read } else { $bhang = $bhang * $cs; # hang is composed of the C read } printf(OVA "%9s %9s %s %5d %5d %.2f %.2f\n", $rn, $cn, ($rf == $cf) ? "N" : "I", $ahang, $bhang, 0.0, 0.0); $olaps++; } } } close(OVA); # Output reads that aligned, didn't align. open(R, "< $readsFile"); open(M, "> $outputPrefix.mapped.fastq"); open(P, "> $outputPrefix.partial.fastq"); open(F, "> $outputPrefix.failed.fastq"); my $mappedReads = 0; my $partialReads = 0; my $failedReads = 0; my @mappedReads; my @orientReads; while (!eof(R)) { my $a = ; my $b = ; my $c = ; my $d = ; my $n; if ($a =~ m/\@(\S+)\s*/) { $n = $1; } else { chomp; die "Failed to parse read name from '$a'\n"; } #die "Failed to find UIDtoNAME for UID '$n'\n" if (!exists($UIDtoNAME{$n})); # #$a = "\@$UIDtoNAME{$n}\n"; if ($goodAligns{$n} > 0) { $mappedReads++; print M "$a$b$c$d"; die "No position for '$n'\n" if (!exists($positions{$n})); push @mappedReads, "$positions{$n}\0$a$b$c$d"; push @orientReads, "$positions{$n}\0$a$b$c$d"; } elsif ($totalAligns{$n} > 0) { $partialReads++; print P "$a$b$c$d"; } else { $failedReads++; print F "$a$b$c$d"; } } close(F); close(P); close(M); close(R); @mappedReads = sort { $a <=> $b } @mappedReads; @orientReads = sort { $a <=> $b } @orientReads; open(A, "> $outputPrefix.mapped.ordered.fastq"); foreach my $a (@mappedReads) { my ($idx,$dat) = split '\0', $a; print A "$dat"; } close(A); open(A, "> $outputPrefix.mapped.ordered.oriented.fastq"); foreach my $a (@orientReads) { my ($idx,$dat) = split '\0', $a; print A "$dat"; } close(A); print STDERR "RESULT $outputPrefix ", scalar(keys %readLength), " input $mappedReads mapped $partialReads partial $failedReads unaligned reads, $totalAligns aligns, $goodAligns good aligns, $olaps overlaps.\n"; canu-1.3/src/overlapInCore-analysis/infer-olaps-from-pairwise-blasr.pl000066400000000000000000000106261272335422300261320ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Convert BLASR default output to OVL overlaps: aIID bIID [I|N] aHang bHang error error_corrected die "Don't use BLASR default output; use SAM format.\n" my %IDmap; # BLASR is not reporting symmetric overlaps. We need to keep track of which overlaps we have reported. my %reported; open(F, "< $ARGV[0]") or die "Failed to open '$ARGV[0]' for reading sequence names.\n"; while () { my ($uid, $iid, $name) = split '\s+', $_; $IDmap{$name} = $iid; } close(F); while () { my @v = split '\s+', $_; $_ = join '\t', $_; # The first read seems to have a sub-read range appended. if ($v[0] =~ m/^(.*)\/\d+_\d+$/) { $v[0] = $1; } my $aiid; my $biid; # Or it might have UID,IID. If this is ALWAYS the case, we don't need the uid map, yay! if ($v[0] =~ m/^\d+,(\d+)$/) { $aiid = $1; } else { $aiid = $IDmap{$v[0]}; } if ($v[1] =~ m/^\d+,(\d+)$/) { $biid = $1; } else { $biid = $IDmap{$v[1]}; } my $ni = ($v[8] == 0) ? "N" : "I"; my $error = 100.0 - $v[3]; my $ecorr = 100.0 - $v[3]; if ((exists($reported{"$aiid-$biid"}) || (exists($reported{"$biid-$aiid"})))) { next; } $reported{"$aiid-$biid"} = 0; # Argh! Do NOT flip coords if reversed. #($v[9], $v[10]) = ($v[10], $v[9]) if ($v[8] == 1); die "No A iid found for '$v[0]'\n" if (!defined($aiid)); die "No B iid found for '$v[1]'\n" if (!defined($biid)); next if ($aiid == $biid); #next if ($aiid < $biid); die "First read is flipped\n$_\n" if ($v[4] != 0); my $a1 = $v[5]; # Amount unaligned on left of first my $b1 = $v[7] - $v[6]; my $a2 = $v[9]; my $b2 = $v[11] - $v[10]; #print "'$v[7]' '$v[8]' '$v[9]' '$v[10]'\n"; #print "a1 $a1 from [5] $v[5]\n"; #print "b1 $b1 from [7]-[6] $v[7] - $v[6]\n"; #print "a2 $a2 from [9] $v[9]\n"; #print "b2 $b2 from [11]-[10] $v[11] - $v[10]\n"; my $fl = $v[8]; my $ahang = 0; my $bhang = 0; my $label = ""; # Handle A contained in B if (($a1 == 0) && ($b1 == 0)) { if ($fl == 0) { $ahang = -$a2; $bhang = $b2; $label = "AinBf"; } else { $ahang = -$b2; $bhang = $a2; $label = "AinBr"; } } # Handle B contained in A elsif (($a2 == 0) && ($b2 == 0)) { if ($fl == 0) { $ahang = $a1; $bhang = -$b1; $label = "BinAf"; } else { $ahang = $b1; $bhang = -$a1; $label = "BinAr"; } } # Handle a dovetail off the left end of A elsif (($a1 == 0) && ($b2 == 0) && ($fl == 0)) { $ahang = -$a2; $bhang = -$b1; $label = "BdoveAf"; } elsif (($a1 == 0) && ($a2 == 0) && ($fl == 1)) { $ahang = -$b2; $bhang = -$b1; $label = "BdoveAr"; } # Handle dovetail off the right end of A elsif (($b1 == 0) && ($a2 == 0) && ($fl == 0)) { $ahang = $a1; $bhang = $b2; $label = "AdoveBf"; } elsif (($b1 == 0) && ($b2 == 0) && ($fl == 1)) { $ahang = $a1; $bhang = $a2; $label = "AdoveBr"; } # All the rest aren't valid overlaps. else { next; $label = "INVALID"; } # HACK! $error = 0.01; $ecorr = 0.01; #print "$aiid\t$biid\t$ni\t$ahang\t$bhang\t$error\t$ecorr\t$label\t$_"; print "$aiid\t$biid\t$ni\t$ahang\t$bhang\t$error\t$ecorr\n"; } canu-1.3/src/overlapInCore-analysis/infer-olaps-from-pairwise-coords.pl000066400000000000000000000116511272335422300263170ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # cd /work/blasr-overlapper/JUNKTEST3/CAordered # time sh /work/scripts/blasr-overlaps.sh BLASROVERLAP.fasta BLASROVERLAP.fasta BLASROVERLAP #7043.059u 16.347s 22:09.56 530.9% 6573+816k 11+192523io 11881pf+0w # # NMAME to IID MAP # my %NAMEtoIID; open(F, "< CAordered/test.gkpStore.fastqUIDmap") or die "Failed to open 'CAordered/test.gkpStore.fastqUIDmap'\n"; while () { my @v = split '\s+', $_; #$IIDtoNAME{$v[1]} = $v[2]; $NAMEtoIID{$v[2]} = $v[1]; } close(F); # # READ LENGTHS # my @readLength; my %contained; open(F, "gatekeeper -dumpfragments -tabular BL/test.gkpStore |"); while () { my @v = split '\s+', $_; $readLength[$v[1]] = $v[9]; } close(F); open(F, "gatekeeper -dumpfragments -tabular CAordered/test.gkpStore |"); while () { my @v = split '\s+', $_; die "IID $v[1] BL $readLength[$v[1]] != CA $v[9]\n" if ($readLength[$v[1]] != $v[9]); } close(F); # # # my %reported; open(F, "< BO/PAIRS.blasr.sam.coords") or die "Failed to open 'BO/PAIRS.blasr.sam.coords'\n"; open(O, "> BO/PAIRS.blasr.sam.ova"); $_ = ; $_ = ; $_ = ; $_ = ; while () { my @v = split '\s+', $_; my $aIID = $NAMEtoIID{$v[9]}; my $bIID = $NAMEtoIID{$v[10]}; next if (!defined($aIID)); # Extra overlaps are not an error. next if (!defined($bIID)); die "undef for a $v[9]\n" if (!defined($aIID)); # Extra overlaps are an error. die "undef for b $v[10]\n" if (!defined($bIID)); next if ($aIID == $bIID); next if (exists($reported{"$aIID-$bIID"})); $reported{"$aIID-$bIID"}++; $reported{"$bIID-$aIID"}++; my $l1 = $readLength[$aIID]; my $a1 = $v[0] - 1; # Amount unaligned on left of first my $b1 = $l1 - $v[1]; # Amount unaligned on right of first my $ori; my $l2 = $readLength[$bIID]; my $a2; my $b2; if ($v[2] < $v[3]) { $ori = "N"; $a2 = $v[2] - 1; $b2 = $l2 - $v[3]; } else { $ori = "I"; $a2 = $v[3] - 1; $b2 = $l2 - $v[2]; } # Extend near global to be global. my $maxTol = 10; $a1 = 0 if ($a1 < $maxTol); $b1 = $l1 if ($l1 < $b1 + $maxTol); $a2 = 0 if ($a2 < $maxTol); $b2 = $l2 if ($l2 < $b2 + $maxTol); my $fl = ($ori eq "I"); my $ahang = 0; my $bhang = 0; my $label = ""; # Handle A contained in B if (($a1 == 0) && ($b1 == 0)) { if ($fl == 0) { $ahang = -$a2; $bhang = $b2; $label = "AinBf"; } else { $ahang = -$b2; $bhang = $a2; $label = "AinBr"; } } # Handle B contained in A elsif (($a2 == 0) && ($b2 == 0)) { if ($fl == 0) { $ahang = $a1; $bhang = -$b1; $label = "BinAf"; } else { $ahang = $b1; $bhang = -$a1; $label = "BinAr"; } } # Handle a dovetail off the left end of A elsif (($a1 == 0) && ($b2 == 0) && ($fl == 0)) { $ahang = -$a2; $bhang = -$b1; $label = "BdoveAf"; } elsif (($a1 == 0) && ($a2 == 0) && ($fl == 1)) { $ahang = -$b2; $bhang = -$b1; $label = "BdoveAr"; } # Handle dovetail off the right end of A elsif (($b1 == 0) && ($a2 == 0) && ($fl == 0)) { $ahang = $a1; $bhang = $b2; $label = "AdoveBf"; } elsif (($b1 == 0) && ($b2 == 0) && ($fl == 1)) { $ahang = $a1; $bhang = $a2; $label = "AdoveBr"; } # All the rest aren't valid overlaps. else { next; $label = "INVALID"; } my $error = 100 - $v[8]; my $ecorr = 100 - $v[8]; #print "$aIID\t$bIID\t$ni\t$ahang\t$bhang\t$error\t$ecorr\t$label\t$_"; print O "$aIID\t$bIID\t$ori\t$ahang\t$bhang\t$error\t$ecorr\n"; } close(F); close(O); system("convertOverlap -ovl < BO/PAIRS.blasr.sam.ova > BO/PAIRS.blasr.sam.ovb"); system("overlapStoreBuild -g CAordered/test.gkpStore -o blasr-pairs.ovlStore -F 1 BO/PAIRS.blasr.sam.ovb"); canu-1.3/src/overlapInCore-analysis/infer-ovl-from-genomic-blasr.pl000066400000000000000000000105351272335422300254110ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Convert BLASR default output to OVL overlaps: aIID bIID [I|N] aHang bHang error error_corrected my %IDmap; # BLASR is not reporting symmetric overlaps. We need to keep track of which overlaps we have reported. my %reported; open(F, "< $ARGV[0]") or die "Failed to open '$ARGV[0]' for reading sequence names.\n"; while () { my ($uid, $iid, $name) = split '\s+', $_; $IDmap{$name} = $iid; } close(F); while () { my @v = split '\s+', $_; $_ = join '\t', $_; # The first read seems to have a sub-read range appended. if ($v[0] =~ m/^(.*)\/\d+_\d+$/) { $v[0] = $1; } my $aiid; my $biid; # Or it might have UID,IID. If this is ALWAYS the case, we don't need the uid map, yay! if ($v[0] =~ m/^\d+,(\d+)$/) { $aiid = $1; } else { $aiid = $IDmap{$v[0]}; } if ($v[1] =~ m/^\d+,(\d+)$/) { $biid = $1; } else { $biid = $IDmap{$v[1]}; } my $ni = ($v[8] == 0) ? "N" : "I"; my $error = 100.0 - $v[3]; my $ecorr = 100.0 - $v[3]; if ((exists($reported{"$aiid-$biid"}) || (exists($reported{"$biid-$aiid"})))) { next; } $reported{"$aiid-$biid"} = 0; # Argh! Do NOT flip coords if reversed. #($v[9], $v[10]) = ($v[10], $v[9]) if ($v[8] == 1); die "No A iid found for '$v[0]'\n" if (!defined($aiid)); die "No B iid found for '$v[1]'\n" if (!defined($biid)); next if ($aiid == $biid); #next if ($aiid < $biid); die "First read is flipped\n$_\n" if ($v[4] != 0); my $a1 = $v[5]; # Amount unaligned on left of first my $b1 = $v[7] - $v[6]; my $a2 = $v[9]; my $b2 = $v[11] - $v[10]; #print "'$v[7]' '$v[8]' '$v[9]' '$v[10]'\n"; #print "a1 $a1 from [5] $v[5]\n"; #print "b1 $b1 from [7]-[6] $v[7] - $v[6]\n"; #print "a2 $a2 from [9] $v[9]\n"; #print "b2 $b2 from [11]-[10] $v[11] - $v[10]\n"; my $fl = $v[8]; my $ahang = 0; my $bhang = 0; my $label = ""; # Handle A contained in B if (($a1 == 0) && ($b1 == 0)) { if ($fl == 0) { $ahang = -$a2; $bhang = $b2; $label = "AinBf"; } else { $ahang = -$b2; $bhang = $a2; $label = "AinBr"; } } # Handle B contained in A elsif (($a2 == 0) && ($b2 == 0)) { if ($fl == 0) { $ahang = $a1; $bhang = -$b1; $label = "BinAf"; } else { $ahang = $b1; $bhang = -$a1; $label = "BinAr"; } } # Handle a dovetail off the left end of A elsif (($a1 == 0) && ($b2 == 0) && ($fl == 0)) { $ahang = -$a2; $bhang = -$b1; $label = "BdoveAf"; } elsif (($a1 == 0) && ($a2 == 0) && ($fl == 1)) { $ahang = -$b2; $bhang = -$b1; $label = "BdoveAr"; } # Handle dovetail off the right end of A elsif (($b1 == 0) && ($a2 == 0) && ($fl == 0)) { $ahang = $a1; $bhang = $b2; $label = "AdoveBf"; } elsif (($b1 == 0) && ($b2 == 0) && ($fl == 1)) { $ahang = $a1; $bhang = $a2; $label = "AdoveBr"; } # All the rest aren't valid overlaps. else { next; $label = "INVALID"; } # HACK! $error = 0.01; $ecorr = 0.01; #print "$aiid\t$biid\t$ni\t$ahang\t$bhang\t$error\t$ecorr\t$label\t$_"; print "$aiid\t$biid\t$ni\t$ahang\t$bhang\t$error\t$ecorr\n"; } canu-1.3/src/overlapInCore/000077500000000000000000000000001272335422300156455ustar00rootroot00000000000000canu-1.3/src/overlapInCore/liboverlap/000077500000000000000000000000001272335422300200045ustar00rootroot00000000000000canu-1.3/src/overlapInCore/liboverlap/Binomial_Bound.C000066400000000000000000000123301272335422300227700ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-FEB-09 to 2015-JUN-02 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-12 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "Binomial_Bound.H" #include "gkStore.H" #undef COMPUTE_IN_LOG_SPACE // Determined by EDIT_DIST_PROB_BOUND #define NORMAL_DISTRIB_THOLD 3.62 // Probability limit to "band" edit-distance calculation // Determines NORMAL_DISTRIB_THOLD #define EDIT_DIST_PROB_BOUND 1e-4 // Return the smallest n >= Start s.t. // prob [>= e errors in n binomial trials (p = error prob)] > EDIT_DIST_PROB_BOUND // int Binomial_Bound(int e, double p, int Start) { double Normal_Z, Mu_Power, Factorial, Poisson_Coeff; double q, Sum, P_Power, Q_Power, X; int k, n, Bin_Coeff, Ct; q = 1.0 - p; if (Start < e) Start = e; for (n = Start; n < AS_MAX_READLEN; n ++) { if (n <= 35) { Sum = 0.0; Bin_Coeff = 1; Ct = 0; P_Power = 1.0; Q_Power = pow (q, n); for (k = 0; k < e && 1.0 - Sum > EDIT_DIST_PROB_BOUND; k ++) { X = Bin_Coeff * P_Power * Q_Power; Sum += X; Bin_Coeff *= n - Ct; Bin_Coeff /= ++ Ct; P_Power *= p; Q_Power /= q; } if (1.0 - Sum > EDIT_DIST_PROB_BOUND) return(n); } else { Normal_Z = (e - 0.5 - n * p) / sqrt (n * p * q); if (Normal_Z <= NORMAL_DISTRIB_THOLD) return n; #ifndef COMPUTE_IN_LOG_SPACE Sum = 0.0; Mu_Power = 1.0; Factorial = 1.0; Poisson_Coeff = exp (- n * p); for (k = 0; k < e; k ++) { Sum += Mu_Power * Poisson_Coeff / Factorial; Mu_Power *= n * p; Factorial *= k + 1; } #else Sum = 0.0; Mu_Power = 0.0; Factorial = 0.0; Poisson_Coeff = - n * p; for (k = 0; k < e; k ++) { Sum += exp(Mu_Power + Poisson_Coeff - Factorial); Mu_Power += log(n * p); Factorial = lgamma(k + 1); } #endif if (1.0 - Sum > EDIT_DIST_PROB_BOUND) return(n); } } return(AS_MAX_READLEN); } void Initialize_Match_Limit(int32 *ml, double maxErate, int32 maxErrors) { int32 e = 0; int32 s = 1; int32 l = MIN(maxErrors, 2000); // Compute the first 2000 values; set to maxErrors to do no estimation // The number of errors that are ignored in setting probability bound for terminating alignment // extensions in edit distance calculations int32 ERRORS_FOR_FREE = 1; // Free errors. while (e <= ERRORS_FOR_FREE) ml[e++] = 0; // Compute the actual limits. This is _VERY_ expensive for longer reads. BITS=17 is about all // it can support. #ifdef DUMP_MATCH_LIMIT l = maxErrors; // For showing deviations #endif while (e < l) { s = Binomial_Bound(e - ERRORS_FOR_FREE, maxErate, s); ml[e] = s - 1; assert(ml[e] >= ml[e-1]); //if ((e % 100) == 0) // fprintf(stderr, " %8.4f%% - %8d / %8d\r", 100.0 * e / maxErrors, e, maxErrors); e++; } // Estimate the remaining limits. using a linear function based on a precomputed slope. // prefixEditDistance-matchLimitGenerate computes the data values for a bunch of error rates. // These are used to compute the slope of a line from the [2000] point through the [max] point. // These slopes fit, almost exactly, an a/x+b curve, and that curve is used to compute the slope // for any error rate. // #if AS_MAX_READLEN_BITS == 17 double sl = 0.962830901135531 / maxErate + 0.096810267016486; #endif #if AS_MAX_READLEN_BITS == 18 double sl = 0.964368146781421 / maxErate + 0.118101522100597; #endif #if AS_MAX_READLEN_BITS == 19 double sl = 0.963823337297648 / maxErate + 0.156091528250625; #endif #if AS_MAX_READLEN_BITS == 20 double sl = 0.971023157863311 / maxErate + 0.154425731746994; #endif #if AS_MAX_READLEN_BITS == 21 double sl = 0.982064188397525 / maxErate + 0.067835741959926; #endif #if AS_MAX_READLEN_BITS == 22 double sl = 0.986446300363063 / maxErate + 0.052358358862826; #endif // And the first value. double vl = ml[e-1] + sl; while (e < maxErrors) { ml[e] = (int32)ceil(vl); vl += sl; e++; } #ifdef DUMP_MATCH_LIMIT FILE *F = fopen("values-new.dat", "w"); for (int32 e=0; e 0) { bot[bot_len++] = '-'; i++; nBgap++; } else { bot[bot_len++] = b[j++]; } } while (j < b_len && i < a_len) { bot[bot_len++] = b[j++]; i++; } bot[bot_len] = '\0'; } uint32 diffs = 0; for (int32 i=0; (i < top_len) || (i < bot_len); i += DISPLAY_WIDTH) { fprintf(stderr, "%d\n", i); fprintf(stderr, "A: "); for (int32 j=0; (j < DISPLAY_WIDTH) && (i+j < top_len); j++) putc(top[i+j], stderr); fprintf(stderr, "\n"); fprintf(stderr, "B: "); for (int32 j=0; (j < DISPLAY_WIDTH) && (i+j < bot_len); j++) putc(bot[i+j], stderr); fprintf(stderr, "\n"); fprintf(stderr, " "); for (int32 j=0; (jStart - 1; int32 S_Right_Begin = Match->Start + Match->Len; int32 S_Right_Len = S_Len - S_Right_Begin; int32 T_Left_Begin = Match->Offset - 1; int32 T_Right_Begin = Match->Offset + Match->Len; int32 T_Right_Len = T_Len - T_Right_Begin; int32 Total_Olap = (min(Match->Start, Match->Offset) + Match->Len + min(S_Right_Len, T_Right_Len)); int32 Error_Limit = Error_Bound[Total_Olap]; #ifdef DEBUG fprintf(stderr, "prefixEditDistance::Extend_Alignment()-- limit olap of %u bases to %u errors - %f%%\n", Total_Olap, Error_Limit, 100.0 * Error_Limit / Total_Olap); fprintf(stderr, "prefixEditDistance::Extend_Alignment()-- S: %d-%d and %d-%d T: %d-%d and %d-%d\n", 0, S_Left_Begin, S_Right_Begin, S_Right_Begin + S_Right_Len, 0, T_Left_Begin, T_Right_Begin, T_Right_Begin + T_Right_Len); #endif Left_Delta_Len = 0; Right_Delta_Len = 0; bool invertLeftDeltas = false; bool invertRightDeltas = false; if ((S_Right_Len == 0) || (T_Right_Len == 0)) { S_Hi = 0; T_Hi = 0; rMatchToEnd = true; } else if (S_Right_Len <= T_Right_Len) { Right_Errors = forward(S + S_Right_Begin, S_Right_Len, T + T_Right_Begin, T_Right_Len, Error_Limit, S_Hi, T_Hi, rMatchToEnd); for (int32 i=0; i 0) { if (Right_Delta[0] > 0) Left_Delta[Left_Delta_Len++] = -(Right_Delta[0] + Leftover + Match->Len); else Left_Delta[Left_Delta_Len++] = -(Right_Delta[0] - Leftover - Match->Len); } // WHY?! Does this mean the invesion on the forward() calls is backwards? for (int32 i=1; i 0; k--) { assert(Edit_Array_Lazy[k] != NULL); from = d; max = 1 + Edit_Array_Lazy[k - 1][d]; if ((j = Edit_Array_Lazy[k - 1][d - 1]) > max) { from = d - 1; max = j; } if ((j = 1 + Edit_Array_Lazy[k - 1][d + 1]) > max) { from = d + 1; max = j; } if (from == d - 1) { Delta_Stack[Right_Delta_Len++] = max - last - 1; d--; last = Edit_Array_Lazy[k - 1][from]; } else if (from == d + 1) { Delta_Stack[Right_Delta_Len++] = last - (max - 1); d++; last = Edit_Array_Lazy[k - 1][from]; } } Delta_Stack[Right_Delta_Len++] = last + 1; k = 0; for (i = Right_Delta_Len - 1; i > 0; i--) Right_Delta[k++] = abs (Delta_Stack[i]) * Sign (Delta_Stack[i - 1]); Right_Delta_Len--; } // Return the minimum number of changes (inserts, deletes, replacements) // needed to match string A[0 .. (m-1)] with a prefix of string // T[0 .. (n-1)] if it's not more than Error_Limit . // If no match, return the number of errors for the best match // up to a branch point. // Put delta description of alignment in Right_Delta and set // Right_Delta_Len to the number of entries there if it's a complete // match. // Set A_End and T_End to the rightmost positions where the // alignment ended in A and T , respectively. // Set Match_To_End true if the match extended to the end // of at least one string; otherwise, set it false to indicate // a branch point. int32 prefixEditDistance::forward(char *A, int32 m, char *T, int32 n, int32 Error_Limit, int32 &A_End, int32 &T_End, bool &Match_To_End) { double Score; int Max_Score_Len = 0, Max_Score_Best_d = 0, Max_Score_Best_e = 0; int Best_d, Best_e, From, Last, Longest, Max, Row; int d, e, i, j, k; assert (m <= n); Best_d = Best_e = Longest = 0; Right_Delta_Len = 0; for (Row = 0; Row < m && (A[Row] == T[Row] || A[Row] == 'n' || T[Row] == 'n'); Row++) ; if (Edit_Array_Lazy[0] == NULL) Allocate_More_Edit_Space(); Edit_Array_Lazy[0][0] = Row; if (Row == m) { // Exact match A_End = T_End = m; Match_To_End = TRUE; #ifdef DEBUG fprintf(stderr, "forward()- exact match\n"); #endif return 0; } int32 Left = 0; int32 Right = 0; double Max_Score = 0.0; for (e = 1; e <= Error_Limit; e++) { if (Edit_Array_Lazy[e] == NULL) Allocate_More_Edit_Space(); Left = MAX (Left - 1, -e); Right = MIN (Right + 1, e); Edit_Array_Lazy[e - 1][Left ] = -2; Edit_Array_Lazy[e - 1][Left - 1] = -2; Edit_Array_Lazy[e - 1][Right ] = -2; Edit_Array_Lazy[e - 1][Right + 1] = -2; for (d = Left; d <= Right; d++) { Row = 1 + Edit_Array_Lazy[e - 1][d]; if ((j = Edit_Array_Lazy[e - 1][d - 1]) > Row) Row = j; if ((j = 1 + Edit_Array_Lazy[e - 1][d + 1]) > Row) Row = j; while (Row < m && Row + d < n && (A[Row] == T[Row + d] || A[Row] == 'n' || T[Row + d] == 'n')) Row++; Edit_Array_Lazy[e][d] = Row; if (Row == m || Row + d == n) { // Check for branch point here caused by uneven distribution of errors Score = Row * Branch_Match_Value - e; // Assumes Branch_Match_Value - Branch_Error_Value == 1.0 int32 Tail_Len = Row - Max_Score_Len; bool abort = false; double slope = (double)(Max_Score - Score) / Tail_Len; if ((doingPartialOverlaps == true) && (Score < Max_Score)) abort = true; #ifdef DEBUG fprintf(stderr, "prefixEditDistance()-- e=%d MIN=%d Tail_Len=%d Max_Score=%d Score=%d slope=%f SLOPE=%f\n", e, MIN_BRANCH_END_DIST, Tail_Len, Max_Score, Score, slope, MIN_BRANCH_TAIL_SLOPE); #endif if ((e > MIN_BRANCH_END_DIST / 2) && (Tail_Len >= MIN_BRANCH_END_DIST) && (slope >= MIN_BRANCH_TAIL_SLOPE)) abort = true; if (abort) { A_End = Max_Score_Len; T_End = Max_Score_Len + Max_Score_Best_d; Set_Right_Delta (Max_Score_Best_e, Max_Score_Best_d); Match_To_End = FALSE; #ifdef DEBUG fprintf(stderr, "forward()- ABORT alignment\n"); #endif return(Max_Score_Best_e); } // Force last error to be mismatch rather than insertion if ((Row == m) && (1 + Edit_Array_Lazy[e - 1][d + 1] == Edit_Array_Lazy[e][d]) && (d < Right)) { d++; Edit_Array_Lazy[e][d] = Edit_Array_Lazy[e][d - 1]; } A_End = Row; // One past last align position T_End = Row + d; Set_Right_Delta (e, d); Match_To_End = TRUE; #ifdef DEBUG fprintf(stderr, "forward()- END alignment\n"); #endif return(e); } } while ((Left <= Right) && (Left < 0) && (Edit_Array_Lazy[e][Left] < Edit_Match_Limit[e])) Left++; if (Left >= 0) while ((Left <= Right) && (Edit_Array_Lazy[e][Left] + Left < Edit_Match_Limit[e])) Left++; if (Left > Right) { #ifdef DEBUG fprintf(stderr, "reverse()- Left=%d Right=%d BREAK\n", Left, Right); #endif break; } while ((Right > 0) && (Edit_Array_Lazy[e][Right] + Right < Edit_Match_Limit[e])) Right--; if (Right <= 0) while (Edit_Array_Lazy[e][Right] < Edit_Match_Limit[e]) Right--; assert (Left <= Right); for (d = Left; d <= Right; d++) if (Edit_Array_Lazy[e][d] > Longest) { Best_d = d; Best_e = e; Longest = Edit_Array_Lazy[e][d]; } Score = Longest * Branch_Match_Value - e; // Assumes Branch_Match_Value - Branch_Error_Value == 1.0 if (Score > Max_Score) { Max_Score = Score; Max_Score_Len = Longest; Max_Score_Best_d = Best_d; Max_Score_Best_e = Best_e; } } #ifdef DEBUG fprintf(stderr, "forward()- return e=%d Error_Limit=%d\n", e, Error_Limit); #endif A_End = Max_Score_Len; T_End = Max_Score_Len + Max_Score_Best_d; Set_Right_Delta (Max_Score_Best_e, Max_Score_Best_d); Match_To_End = FALSE; return Max_Score_Best_e; } canu-1.3/src/overlapInCore/liboverlap/prefixEditDistance-matchLimit.sh000066400000000000000000000007461272335422300262160ustar00rootroot00000000000000#!/bin/sh bgn=`expr $SGE_TASK_ID + 0` end=`expr $SGE_TASK_ID + 1` ./bin-bound-test $bgn $end for er in 0100 0200 0300 0400 0500 0600 0700 0800 0900 1000 \ 1100 1200 1300 1400 1500 1600 1700 1800 1900 2000 \ 2100 2200 2300 2400 2500 2600 2700 2800 2900 3000 \ 3100 3200 3300 3400 3500 3600 3700 3800 3900 4000 \ 4100 4200 4300 4400 4500 4600 4700 4800 4900 5000 ; do qsub -cwd -j y -o /dev/null -b y ./prefixEditDistance-matchLimit $er done canu-1.3/src/overlapInCore/liboverlap/prefixEditDistance-matchLimitGenerate.C000066400000000000000000000160371272335422300274410ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-JUN-02 to 2015-AUG-14 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-FEB-05 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-FEB-15 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "Binomial_Bound.H" #ifndef BROKEN_CLANG_OpenMP #include #endif // To use: // // Set read length in gkStore.H. Run this "50 5000 50" to compute data from 0.50% to 50.00% // in steps of 0.50%. It will write three output files with the data: // *.C - for inclusion in programs // *.bin - binary dump of the array // *.dat - ascii integer dump of the array // // WARNING! BITS=21 needs about 40 CPU hours. (5000 took 2:40; 3800 took 1:28 // // The *.dat also contains the slope of the line from [0] to [i] and from [i] to [max] for each i. // // To generate the slope parameters used in Binomial_Bound.C (BITS=20 and BITS=21 are expensive): // // prefixEditDistance-matchLimitGenerate 100 5000 100 // // grep '^2000 ' *21/*dat | sed 's/.dat:/ /' | sed 's/prefixEditDistance-matchLimitData-BITS=[0-9][0-9]\/prefixEditDistance-matchLimit-//' > slopes // // gnuplot: // f(x) = a/x+b // fit f(x) 'slopes' using 1:5 via a,b // plot 'slopes' using 1:5 with lines, f(x) // show var // plug a and b into Binomial_Bound.C // int main(int argc, char **argv) { int32 minEvalue = 0; int32 maxEvalue = 0; int32 step = 1; char D[FILENAME_MAX]; char O[FILENAME_MAX]; if (argc == 2) { minEvalue = atoi(argv[1]); maxEvalue = minEvalue; } else if (argc == 3) { minEvalue = atoi(argv[1]); maxEvalue = atoi(argv[2]); } else if (argc == 4) { minEvalue = atoi(argv[1]); maxEvalue = atoi(argv[2]); step = atoi(argv[3]); } else { fprintf(stderr, "usage: %s minEvalue [maxEvalue [step]]\n", argv[0]); fprintf(stderr, " computes overlapper probabilities for minEvalue <= eValue <= maxEvalue'\n"); fprintf(stderr, " eValue 100 == 0.01 fraction error == 1%% error\n"); exit(1); } fprintf(stderr, "Computing Edit_Match_Limit data for reads of length %ubp (bits = %u).\n", AS_MAX_READLEN, AS_MAX_READLEN_BITS); sprintf(D, "prefixEditDistance-matchLimitData-BITS=%01d", AS_MAX_READLEN_BITS); AS_UTL_mkdir(D); #pragma omp parallel for schedule(dynamic, 1) for (int32 evalue=maxEvalue; evalue>=minEvalue; evalue -= step) { char N[FILENAME_MAX]; // Local to this thread! double erate = evalue / 10000.0; int32 start = 1; int32 MAX_ERRORS = (1 + (int) (erate * AS_MAX_READLEN)); int32 ERRORS_FOR_FREE = 1; int32 *starts = new int32 [MAX_ERRORS + 1]; memset(starts, 0, sizeof(int32) * (MAX_ERRORS + 1)); sprintf(N, "%s/prefixEditDistance-matchLimit-%04d.bin", D, evalue); if (AS_UTL_fileExists(N)) { fprintf(stderr, "eValue %04d -- eRate %6.4f -- %7.4f%% error -- %8d values -- thread %2d - LOAD\n", evalue, erate, erate * 100.0, MAX_ERRORS, omp_get_thread_num()); errno = 0; FILE *F = fopen(N, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", N, strerror(errno)), exit(1); int32 me = 0; double er = 0.0; fread(&me, sizeof(int32), 1, F); fread(&er, sizeof(double), 1, F); fread( starts, sizeof(int32), MAX_ERRORS, F); assert(me == MAX_ERRORS); assert(er == erate); fclose(F); } else { fprintf(stderr, "eValue %04d -- eRate %6.4f -- %7.4f%% error -- %8d values -- thread %2d - COMPUTE\n", evalue, erate, erate * 100.0, MAX_ERRORS, omp_get_thread_num()); for (int32 e=ERRORS_FOR_FREE + 1; e 0; k--) { assert(Edit_Array_Lazy[k] != NULL); from = d; max = 1 + Edit_Array_Lazy[k - 1][d]; if ((j = Edit_Array_Lazy[k - 1][d - 1]) > max) { from = d - 1; max = j; } if ((j = 1 + Edit_Array_Lazy[k - 1][d + 1]) > max) { from = d + 1; max = j; } if (from == d - 1) { Left_Delta[Left_Delta_Len++] = max - last - 1; d--; last = Edit_Array_Lazy[k - 1][from]; } else if (from == d + 1) { Left_Delta[Left_Delta_Len++] = last - (max - 1); d++; last = Edit_Array_Lazy[k - 1][from]; } } leftover = last; // Don't allow first delta to be +1 or -1 assert (Left_Delta_Len == 0 || Left_Delta[0] != -1); // BPW - The original test was Left_Delta_Len>0, but that hit uninitialized data when Len == 1. if (Left_Delta_Len > 1 && Left_Delta[0] == 1 && t_end + t_len > 0) { int i; if (Left_Delta[1] > 0) // <- uninitialized here Left_Delta[0] = Left_Delta[1] + 1; else Left_Delta[0] = Left_Delta[1] - 1; for (i = 2; i < Left_Delta_Len; i++) Left_Delta[i - 1] = Left_Delta[i]; Left_Delta_Len--; t_end--; if (Left_Delta_Len == 0) leftover++; } } // Return the minimum number of changes (inserts, deletes, replacements) // needed to match string A[0 .. (1-m)] right-to-left with a prefix of string // T[0 .. (1-n)] right-to-left if it's not more than Error_Limit . // If no match, return the number of errors for the best match // up to a branch point. // Put delta description of alignment in Left_Delta and set // Left_Delta_Len to the number of entries there. // Set A_End and T_End to the leftmost positions where the // alignment ended in A and T , respectively. // If the alignment succeeds set Leftover to the number of // characters that match after the last Left_Delta entry; // otherwise, set Leftover to zero. // Set Match_To_End true if the match extended to the end // of at least one string; otherwise, set it false to indicate // a branch point. int32 prefixEditDistance::reverse(char *A, int32 m, char *T, int32 n, int32 Error_Limit, int32 &A_End, int32 &T_End, int32 &Leftover, // <- novel bool &Match_To_End) { double Score; int Max_Score_Len = 0, Max_Score_Best_d = 0, Max_Score_Best_e = 0; int Best_d, Best_e, From, Last, Longest, Max, Row; int d, e, j, k; assert (m <= n); Best_d = Best_e = Longest = 0; Left_Delta_Len = 0; for (Row = 0; Row < m && (A[- Row] == T[- Row] || A[- Row] == 'n' || T[- Row] == 'n'); Row++) ; if (Edit_Array_Lazy[0] == NULL) Allocate_More_Edit_Space(); Edit_Array_Lazy[0][0] = Row; if (Row == m) { A_End = T_End = - m; Leftover = m; Match_To_End = TRUE; #ifdef DEBUG fprintf(stderr, "reverse()- exact match\n"); #endif return 0; } int32 Left = 0; int32 Right = 0; double Max_Score = 0.0; for (e = 1; e <= Error_Limit; e++) { if (Edit_Array_Lazy[e] == NULL) Allocate_More_Edit_Space(); Left = MAX (Left - 1, -e); Right = MIN (Right + 1, e); Edit_Array_Lazy[e - 1][Left ] = -2; Edit_Array_Lazy[e - 1][Left - 1] = -2; Edit_Array_Lazy[e - 1][Right ] = -2; Edit_Array_Lazy[e - 1][Right + 1] = -2; for (d = Left; d <= Right; d++) { Row = 1 + Edit_Array_Lazy[e - 1][d]; if ((j = Edit_Array_Lazy[e - 1][d - 1]) > Row) Row = j; if ((j = 1 + Edit_Array_Lazy[e - 1][d + 1]) > Row) Row = j; while (Row < m && Row + d < n && (A[- Row] == T[- Row - d] || A[- Row] == 'n' || T[- Row - d] == 'n')) Row++; Edit_Array_Lazy[e][d] = Row; if (Row == m || Row + d == n) { // Check for branch point here caused by uneven distribution of errors Score = Row * Branch_Match_Value - e; // Assumes Branch_Match_Value - Branch_Error_Value == 1.0 int32 Tail_Len = Row - Max_Score_Len; bool abort = false; double slope = (double)(Max_Score - Score) / Tail_Len; if ((doingPartialOverlaps == true) && (Score < Max_Score)) abort = true; #ifdef DEBUG fprintf(stderr, "prefixEditDistance()-- e=%d MIN=%d Tail_Len=%d Max_Score=%d Score=%d slope=%f SLOPE=%f\n", e, MIN_BRANCH_END_DIST, Tail_Len, Max_Score, Score, slope, MIN_BRANCH_TAIL_SLOPE); #endif if ((e > MIN_BRANCH_END_DIST / 2) && (Tail_Len >= MIN_BRANCH_END_DIST) && (slope >= MIN_BRANCH_TAIL_SLOPE)) abort = true; if (abort) { A_End = - Max_Score_Len; T_End = - Max_Score_Len - Max_Score_Best_d; Set_Left_Delta (Max_Score_Best_e, Max_Score_Best_d, Leftover, T_End, n); Match_To_End = FALSE; #ifdef DEBUG fprintf(stderr, "reverse()- ABORT alignment\n"); #endif return(Max_Score_Best_e); } A_End = - Row; // One past last align position T_End = - Row - d; Set_Left_Delta (e, d, Leftover, T_End, n); Match_To_End = TRUE; #ifdef DEBUG fprintf(stderr, "reverse()- END alignment\n"); #endif return(e); } } while ((Left <= Right) && (Left < 0) && (Edit_Array_Lazy[e][Left] < Edit_Match_Limit[e])) Left++; if (Left >= 0) while ((Left <= Right) && (Edit_Array_Lazy[e][Left] + Left < Edit_Match_Limit[e])) Left++; if (Left > Right) { #ifdef DEBUG fprintf(stderr, "reverse()- Left=%d Right=%d BREAK\n", Left, Right); #endif break; } while ((Right > 0) && (Edit_Array_Lazy[e][Right] + Right < Edit_Match_Limit[e])) Right--; if (Right <= 0) while (Edit_Array_Lazy[e][Right] < Edit_Match_Limit[e]) Right--; assert (Left <= Right); for (d = Left; d <= Right; d++) if (Edit_Array_Lazy[e][d] > Longest) { Best_d = d; Best_e = e; Longest = Edit_Array_Lazy[e][d]; } Score = Longest * Branch_Match_Value - e; // Assumes Branch_Match_Value - Branch_Error_Value == 1.0 if (Score > Max_Score) { Max_Score = Score; Max_Score_Len = Longest; Max_Score_Best_d = Best_d; Max_Score_Best_e = Best_e; } } #ifdef DEBUG fprintf(stderr, "reverse()- return e=%d Error_Limit=%d\n", e, Error_Limit); #endif A_End = - Max_Score_Len; T_End = - Max_Score_Len - Max_Score_Best_d; Set_Left_Delta (Max_Score_Best_e, Max_Score_Best_d, Leftover, T_End, n); Match_To_End = FALSE; return Max_Score_Best_e; } canu-1.3/src/overlapInCore/liboverlap/prefixEditDistance.C000066400000000000000000000070301272335422300236660ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-JUN-03 to 2015-SEP-21 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-FEB-05 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "prefixEditDistance.H" #include "Binomial_Bound.H" prefixEditDistance::prefixEditDistance(bool doingPartialOverlaps_, double maxErate_) { maxErate = maxErate_; doingPartialOverlaps = doingPartialOverlaps_; MAX_ERRORS = (1 + (int)ceil(maxErate * AS_MAX_READLEN)); MIN_BRANCH_END_DIST = 20; MIN_BRANCH_TAIL_SLOPE = ((maxErate > 0.06) ? 1.0 : 0.20); Left_Delta = new int [MAX_ERRORS]; Right_Delta = new int [MAX_ERRORS]; allocated = 3 * MAX_ERRORS * sizeof(int); Delta_Stack = new int [MAX_ERRORS]; Edit_Space_Lazy = new int * [MAX_ERRORS]; Edit_Array_Lazy = new int * [MAX_ERRORS]; memset(Edit_Space_Lazy, 0, sizeof(int *) * MAX_ERRORS); memset(Edit_Array_Lazy, 0, sizeof(int *) * MAX_ERRORS); allocated += MAX_ERRORS * sizeof (int); allocated += MAX_ERRORS * sizeof (int); // Edit_Match_Limit_Allocation = new int32 [MAX_ERRORS + 1]; Edit_Match_Limit = Edit_Match_Limit_Allocation; Initialize_Match_Limit(Edit_Match_Limit_Allocation, maxErate, MAX_ERRORS); for (int32 i=0; i <= AS_MAX_READLEN; i++) { //Error_Bound[i] = (int32) (i * maxErate + 0.0000000000001); Error_Bound[i] = (int32)ceil(i * maxErate); } // Value to add for a match in finding branch points. // // ALH: Note that maxErate also affects what overlaps get found // // ALH: Scoring seems to be unusual: given an alignment // of length l with k mismatches, the score seems to be // computed as l + k * error value and NOT (l-k)*match+k*error // // I.e. letting x := DEFAULT_BRANCH_MATCH_VAL, // the max mismatch fraction p to give a non-negative score // would be p = x/(1-x); conversely, to compute x for a // goal p, we have x = p/(1+p). E.g. // // for p=0.06, x = .06 / (1.06) = .0566038 // for p=0.35, x = .35 / (1.35) = .259259 // for p=0.2, x = .20 / (1.20) = .166667 // for p=0.15, x = .15 / (1.15) = .130435 // // Value was for 6% vs 35% error discrimination. // Converting to integers didn't make it faster. // Corresponding error value is this value minus 1.0 Branch_Match_Value = maxErate / (1 + maxErate); Branch_Error_Value = Branch_Match_Value - 1.0; }; prefixEditDistance::~prefixEditDistance() { delete [] Left_Delta; delete [] Right_Delta; delete [] Delta_Stack; for (uint32 i=0; i 0) - ((a) < 0) ) enum Overlap_t { NONE, LEFT_BRANCH_PT, RIGHT_BRANCH_PT, DOVETAIL }; // the input to Extend_Alignment. struct Match_Node_t { int32 Offset; // To start of exact match in hash-table frag int32 Len; // Of exact match int32 Start; // Of exact match in current (new) frag int32 Next; // Subscript of next match in list }; class prefixEditDistance { public: prefixEditDistance(bool doingPartialOverlaps_, double maxErate_); ~prefixEditDistance(); void Allocate_More_Edit_Space(void); void Set_Right_Delta(int32 e, int32 d); int32 forward(char *A, int32 m, char *T, int32 n, int32 Error_Limit, int32 &A_End, int32 &T_End, bool &Match_To_End); void Set_Left_Delta(int32 e, int32 d, int32 &leftover, int32 &t_end, int32 t_len); int32 reverse(char *A, int32 m, char *T, int32 n, int32 Error_Limit, int32 &A_End, int32 &T_End, int32 &Leftover, // <- novel bool &Match_To_End); Overlap_t Extend_Alignment(Match_Node_t *Match, char *S, int32 S_Len, char *T, int32 T_Len, int32 &S_Lo, int32 &S_Hi, int32 &T_Lo, int32 &T_Hi, int32 &Errors); public: // The four below were global #defines, two depended on the error rate which is now local. // Most errors in any edit distance computation uint32 MAX_ERRORS; // Branch points must be at least this many bases from the end of the fragment to be reported uint32 MIN_BRANCH_END_DIST; // Branch point tails must fall off from the max by at least this rate double MIN_BRANCH_TAIL_SLOPE; double maxErate; bool doingPartialOverlaps; uint64 allocated; int32 Left_Delta_Len; int32 *Left_Delta; int32 Right_Delta_Len; int32 *Right_Delta; int32 *Delta_Stack; int32 **Edit_Space_Lazy; // Array of pointers, if set, it was a new'd allocation int32 **Edit_Array_Lazy; // Array of pointers, some are not new'd allocations // This array [e] is the minimum value of Edit_Array[e][d] // to be worth pursuing in edit-distance computations between reads const int32 *Edit_Match_Limit; int32 *Edit_Match_Limit_Allocation; // The maximum number of errors allowed in a match between reads of length i, // which is i * AS_OVL_ERROR_RATE. int32 Error_Bound[AS_MAX_READLEN + 1]; // Scores of matches and mismatches in alignments. Alignment ends at maximum score. double Branch_Match_Value; double Branch_Error_Value; }; #endif canu-1.3/src/overlapInCore/overlapConvert.C000066400000000000000000000061111272335422300207610ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-FEB-11 to 2015-JUN-25 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include using namespace std; int main(int argc, char **argv) { char *gkpStoreName = NULL; gkStore *gkpStore = NULL; ovOverlapDisplayType dt = ovOverlapAsCoords; vector files; int32 arg = 1; int32 err = 0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-coords") == 0) { dt = ovOverlapAsCoords; } else if (strcmp(argv[arg], "-hangs") == 0) { dt = ovOverlapAsHangs; } else if (strcmp(argv[arg], "-raw") == 0) { dt = ovOverlapAsRaw; } else if (AS_UTL_fileExists(argv[arg])) { files.push_back(argv[arg]); } else { fprintf(stderr, "ERROR: invalid arg '%s'\n", argv[arg]); err++; } arg++; } if ((gkpStoreName == NULL) && (dt == ovOverlapAsCoords)) err++; if ((err) || (files.size() == 0)) { fprintf(stderr, "usage: %s [options] file.ovb[.gz]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " -G gkpStore (needed for -coords, the default)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -coords output coordiantes on reads\n"); fprintf(stderr, " -hangs output hangs on reads\n"); fprintf(stderr, " -raw output raw hangs on reads\n"); if ((gkpStoreName == NULL) && (dt == ovOverlapAsCoords)) fprintf(stderr, "ERROR: -coords mode requires a gkpStore (-G)\n"); if (files.size() == 0) fprintf(stderr, "ERROR: no overlap files supplied\n"); exit(1); } if (gkpStoreName) gkpStore = gkStore::gkStore_open(gkpStoreName); char *ovStr = new char [1024]; for (uint32 ff=0; ffreadOverlap(&ov)) fputs(ov.toString(ovStr, dt, true), stdout); delete of; } delete [] ovStr; gkpStore->gkStore_close(); exit(0); } canu-1.3/src/overlapInCore/overlapConvert.mk000066400000000000000000000007621272335422300212140ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := overlapConvert SOURCES := overlapConvert.C SRC_INCDIRS := .. ../AS_UTL ../stores liboverlap TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/overlapInCore/overlapImport.C000066400000000000000000000142231272335422300206160ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAY-14 to 2015-JUN-25 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-FEB-17 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include "splitToWords.H" #include using namespace std; #define TYPE_NONE 'N' #define TYPE_LEGACY 'L' #define TYPE_COORDS 'C' #define TYPE_HANGS 'H' #define TYPE_RAW 'R' int main(int argc, char **argv) { char *gkpStoreName = NULL; gkStore *gkpStore = NULL; char *ovlFileName = NULL; char *ovlStoreName = NULL; char inType = TYPE_NONE; vector files; int32 arg = 1; int32 err = 0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-o") == 0) { ovlFileName = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { ovlStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-legacy") == 0) { inType = TYPE_LEGACY; } else if (strcmp(argv[arg], "-coords") == 0) { fprintf(stderr, "-coords not implemented.\n"), exit(1); inType = TYPE_COORDS; } else if (strcmp(argv[arg], "-hangs") == 0) { fprintf(stderr, "-hangs not implemented.\n"), exit(1); inType = TYPE_HANGS; } else if (strcmp(argv[arg], "-raw") == 0) { inType = TYPE_RAW; } else if ((strcmp(argv[arg], "-") == 0) || (AS_UTL_fileExists(argv[arg]))) { files.push_back(argv[arg]); } else { fprintf(stderr, "ERROR: invalid arg '%s'\n", argv[arg]); err++; } arg++; } if (gkpStoreName == NULL) err++; if (inType == TYPE_NONE) err++; if ((err) || (files.size() == 0)) { fprintf(stderr, "usage: %s [options] ascii-ovl-file-input.[.gz]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, "Required:\n"); fprintf(stderr, " -G name.gkpStore path to valid gatekeeper store\n"); fprintf(stderr, "\n"); fprintf(stderr, "Output options:\n"); fprintf(stderr, " -o file.ovb output file name\n"); fprintf(stderr, " -O name.ovlStore output overlap store"); fprintf(stderr, "\n"); fprintf(stderr, "Format options:\n"); fprintf(stderr, " -legacy 'CA8 overlapStore -d' format\n"); fprintf(stderr, " -coords 'overlapConvert -coords' format (not implemented)\n"); fprintf(stderr, " -hangs 'overlapConvert -hangs' format (not implemented)\n"); fprintf(stderr, " -raw 'overlapConvert -raw' format\n"); fprintf(stderr, "\n"); fprintf(stderr, "Input file can be stdin ('-') or a gz/bz2/xz compressed file.\n"); fprintf(stderr, "\n"); if (gkpStoreName == NULL) fprintf(stderr, "ERROR: need to supply a gkpStore (-G).\n"); if (inType == TYPE_NONE) fprintf(stderr, "ERROR: need to supply a format type (-legacy, -coords, -hangs, -raw).\n"); exit(1); } if (gkpStoreName) gkpStore = gkStore::gkStore_open(gkpStoreName); char *S = new char [1024]; splitToWords W; ovOverlap ov(gkpStore); ovFile *of = (ovlFileName == NULL) ? NULL : new ovFile(ovlFileName, ovFileFullWrite); ovStore *os = (ovlStoreName == NULL) ? NULL : new ovStore(ovlStoreName, gkpStore, ovStoreWrite); for (uint32 ff=0; fffile()); while (!feof(in->file())) { W.split(S); switch (inType) { case TYPE_LEGACY: // Aiid Biid 'I/N' ahang bhang erate erate ov.a_iid = W(0); ov.b_iid = W(1); ov.flipped(W[2][0] == 'I'); ov.a_hang(W(3)); ov.b_hang(W(4)); // Overlap store reports %error, but we expect fraction error. //ov.erate(atof(W[5]); // Don't use the original uncorrected error rate ov.erate(atof(W[6]) / 100.0); break; case TYPE_COORDS: break; case TYPE_HANGS: break; case TYPE_RAW: ov.a_iid = W(0); ov.b_iid = W(1); ov.flipped(W[2][0] == 'I'); ov.dat.ovl.span = W(3); ov.dat.ovl.ahg5 = W(4); ov.dat.ovl.ahg3 = W(5); ov.dat.ovl.bhg5 = W(6); ov.dat.ovl.bhg3 = W(7); ov.erate(atof(W[8]) / 1); ov.dat.ovl.forUTG = false; ov.dat.ovl.forOBT = false; ov.dat.ovl.forDUP = false; for (uint32 i = 9; i < W.numWords(); i++) { ov.dat.ovl.forUTG |= ((W[i][0] == 'U') && (W[i][1] == 'T') && (W[i][2] == 'G')); // Fails if W[i] == "U". ov.dat.ovl.forOBT |= ((W[i][0] == 'O') && (W[i][1] == 'B') && (W[i][2] == 'T')); ov.dat.ovl.forDUP |= ((W[i][0] == 'D') && (W[i][1] == 'U') && (W[i][2] == 'P')); } break; default: break; } if (of) of->writeOverlap(&ov); if (os) os->writeOverlap(&ov); fgets(S, 1024, in->file()); } delete in; } delete os; delete of; delete [] S; gkpStore->gkStore_close(); exit(0); } canu-1.3/src/overlapInCore/overlapImport.mk000066400000000000000000000007601272335422300210440ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := overlapImport SOURCES := overlapImport.C SRC_INCDIRS := .. ../AS_UTL ../stores liboverlap TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/overlapInCore/overlapInCore-Build_Hash_Index.C000066400000000000000000000512651272335422300236610ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVL/AS_OVL_overlap_common.h * src/AS_OVM/overlapInCore-Build_Hash_Index.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-JUL-15 to 2007-NOV-20 * are Copyright 2005,2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2006-MAR-27 to 2006-AUG-21 * are Copyright 2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Art Delcher on 2007-FEB-13 * are Copyright 2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2007-AUG-27 to 2009-JAN-16 * are Copyright 2007,2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2011-MAR-08 to 2011-JUN-17 * are Copyright 2011 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz from 2014-DEC-15 to 2015-AUG-25 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-23 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "overlapInCore.H" #include "AS_UTL_reverseComplement.H" // Add string s as an extra hash table string and return // a single reference to the beginning of it. static String_Ref_t Add_Extra_Hash_String(const char *s) { String_Ref_t ref = 0; String_Ref_t sub = 0; int len; uint32 new_len = Used_Data_Len + G.Kmer_Len; if (Extra_String_Subcount < MAX_EXTRA_SUBCOUNT) { sub = String_Ct + Extra_String_Ct - 1; } else { sub = String_Ct + Extra_String_Ct; if (sub >= String_Start_Size) { uint64 n = max(sub * 1.1, String_Start_Size * 1.5); //fprintf(stderr, "REALLOC String_Start from "F_U64" to "F_U64"\n", String_Start_Size, n); resizeArray(String_Start, String_Start_Size, String_Start_Size, n); } String_Start[sub] = Used_Data_Len; Extra_String_Ct++; Extra_String_Subcount = 0; new_len++; } if (new_len >= Extra_Data_Len) { uint64 n = max(new_len * 1.1, Extra_Data_Len * 1.5); //fprintf(stderr, "REALLOC basesData from "F_U64" to "F_U64"\n", Extra_Data_Len, n); resizeArray(basesData, Extra_Data_Len, Extra_Data_Len, n); } strncpy(basesData + String_Start[sub] + G.Kmer_Len * Extra_String_Subcount, s, G.Kmer_Len + 1); Used_Data_Len = new_len; setStringRefStringNum(ref, sub); if (sub > MAX_STRING_NUM) { fprintf(stderr, "Too many skip kmer strings for hash table.\n"); fprintf(stderr, "Try skipping hopeless check (-z option)\n"); fprintf(stderr, "Exiting\n"); exit (1); } setStringRefOffset(ref, (String_Ref_t)Extra_String_Subcount * (String_Ref_t)G.Kmer_Len); assert(Extra_String_Subcount * G.Kmer_Len < OFFSET_MASK); setStringRefLast(ref, (uint64)1); setStringRefEmpty(ref, TRUELY_ONE); Extra_String_Subcount++; return(ref); } // Mark left/right_end_screened in global String_Info for // ref and everything in its list, if they occur near // enough to the end of the string. static void Mark_Screened_Ends_Single(String_Ref_t ref) { int32 s_num = getStringRefStringNum(ref); int32 len = String_Info[s_num].length; if (getStringRefOffset(ref) < HOPELESS_MATCH) String_Info[s_num].lfrag_end_screened = TRUE; if (len - getStringRefOffset(ref) - G.Kmer_Len + 1 < HOPELESS_MATCH) String_Info[s_num].rfrag_end_screened = TRUE; } static void Mark_Screened_Ends_Chain(String_Ref_t ref) { Mark_Screened_Ends_Single (ref); while (! getStringRefLast(ref)) { ref = nextRef[(String_Start[getStringRefStringNum(ref)] + getStringRefOffset(ref)) / (HASH_KMER_SKIP + 1)]; Mark_Screened_Ends_Single (ref); } } // Set the empty bit to true for the hash table entry // corresponding to string s whose hash key is key . // Also set global String_Info.left/right_end_screened // true if the entry occurs near the left/right end, resp., // of the string in the hash table. If not found, add an // entry to the hash table and mark it empty. static void Hash_Mark_Empty(uint64 key, char * s) { String_Ref_t h_ref; char * t; unsigned char key_check; int64 ct, probe; int64 sub; int i, shift; sub = HASH_FUNCTION (key); key_check = KEY_CHECK_FUNCTION (key); probe = PROBE_FUNCTION (key); ct = 0; do { for (i = 0; i < Hash_Table[sub].Entry_Ct; i ++) if (Hash_Table[sub].Check[i] == key_check) { h_ref = Hash_Table[sub].Entry[i]; t = basesData + String_Start[getStringRefStringNum(h_ref)] + getStringRefOffset(h_ref); if (strncmp (s, t, G.Kmer_Len) == 0) { if (! getStringRefEmpty(Hash_Table[sub].Entry[i])) Mark_Screened_Ends_Chain (Hash_Table[sub].Entry[i]); setStringRefEmpty(Hash_Table[sub].Entry[i], TRUELY_ONE); return; } } assert (i == Hash_Table[sub].Entry_Ct); if (Hash_Table[sub].Entry_Ct < ENTRIES_PER_BUCKET) { // Not found if (G.Use_Hopeless_Check) { Hash_Table[sub].Entry[i] = Add_Extra_Hash_String (s); setStringRefEmpty(Hash_Table[sub].Entry[i], TRUELY_ONE); Hash_Table[sub].Check[i] = key_check; Hash_Table[sub].Entry_Ct ++; Hash_Table[sub].Hits[i] = 0; Hash_Entries ++; shift = HASH_CHECK_FUNCTION (key); Hash_Check_Array[sub] |= (((Check_Vector_t) 1) << shift); } return; } sub = (sub + probe) % HASH_TABLE_SIZE; } while (++ ct < HASH_TABLE_SIZE); fprintf (stderr, "ERROR: Hash table full\n"); assert (FALSE); } // Set Empty bit true for all entries in global Hash_Table // that match a kmer in file Kmer_Skip_File . // Add the entry (and then mark it empty) if it's not in Hash_Table. static void Mark_Skip_Kmers(void) { uint64 key; char line[MAX_LINE_LEN]; int ct = 0; rewind (G.Kmer_Skip_File); while (fgets (line, MAX_LINE_LEN, G.Kmer_Skip_File) != NULL) { int i, len; ct ++; len = strlen (line) - 1; if (line[0] != '>' || line[len] != '\n') { fprintf (stderr, "ERROR: Bad line %d in kmer skip file\n", ct); fputs (line, stderr); exit (1); } if (fgets (line, MAX_LINE_LEN, G.Kmer_Skip_File) == NULL) { fprintf (stderr, "ERROR: Bad line after %d in kmer skip file\n", ct); exit (1); } ct ++; len = strlen (line) - 1; if (len != G.Kmer_Len || line[len] != '\n') { fprintf (stderr, "ERROR: Bad line %d in kmer skip file\n", ct); fputs (line, stderr); exit (1); } line[len] = '\0'; //if ((ct % 200000) == 0) // fprintf(stderr, "Loaded skip %10d '%s'\n", ct/2, line); key = 0; for (i = 0; i < len; i ++) { line[i] = tolower (line[i]); key |= (uint64) (Bit_Equivalent[(int) line[i]]) << (2 * i); } Hash_Mark_Empty (key, line); reverseComplementSequence (line, len); key = 0; for (i = 0; i < len; i ++) key |= (uint64) (Bit_Equivalent[(int) line[i]]) << (2 * i); Hash_Mark_Empty (key, line); } fprintf (stderr, "String_Ct = "F_U64" Extra_String_Ct = "F_U64" Extra_String_Subcount = "F_U64"\n", String_Ct, Extra_String_Ct, Extra_String_Subcount); fprintf (stderr, "Read %d kmers to mark to skip\n", ct / 2); } // Insert Ref with hash key Key into global Hash_Table . // Ref represents string S . static void Hash_Insert(String_Ref_t Ref, uint64 Key, char * S) { String_Ref_t H_Ref; char * T; int Shift; unsigned char Key_Check; int64 Ct, Probe, Sub; int i; Sub = HASH_FUNCTION (Key); Shift = HASH_CHECK_FUNCTION (Key); Hash_Check_Array[Sub] |= (((Check_Vector_t) 1) << Shift); Key_Check = KEY_CHECK_FUNCTION (Key); Probe = PROBE_FUNCTION (Key); Ct = 0; do { for (i = 0; i < Hash_Table[Sub].Entry_Ct; i ++) if (Hash_Table[Sub].Check[i] == Key_Check) { H_Ref = Hash_Table[Sub].Entry[i]; T = basesData + String_Start[getStringRefStringNum(H_Ref)] + getStringRefOffset(H_Ref); if (strncmp (S, T, G.Kmer_Len) == 0) { if (getStringRefLast(H_Ref)) { Extra_Ref_Ct ++; } nextRef[(String_Start[getStringRefStringNum(Ref)] + getStringRefOffset(Ref)) / (HASH_KMER_SKIP + 1)] = H_Ref; Extra_Ref_Ct ++; setStringRefLast(Ref, TRUELY_ZERO); Hash_Table[Sub].Entry[i] = Ref; if (Hash_Table[Sub].Hits[i] < HIGHEST_KMER_LIMIT) Hash_Table[Sub].Hits[i] ++; return; } } if (i != Hash_Table[Sub].Entry_Ct) { fprintf (stderr, "i = %d Sub = " F_S64 " Entry_Ct = %d\n", i, Sub, Hash_Table[Sub].Entry_Ct); } assert (i == Hash_Table[Sub].Entry_Ct); if (Hash_Table[Sub].Entry_Ct < ENTRIES_PER_BUCKET) { setStringRefLast(Ref, TRUELY_ONE); Hash_Table[Sub].Entry[i] = Ref; Hash_Table[Sub].Check[i] = Key_Check; Hash_Table[Sub].Entry_Ct ++; Hash_Entries ++; Hash_Table[Sub].Hits[i] = 1; return; } Sub = (Sub + Probe) % HASH_TABLE_SIZE; } while (++ Ct < HASH_TABLE_SIZE); fprintf (stderr, "ERROR: Hash table full\n"); assert (FALSE); } // Insert string subscript i into the global hash table. // Sequence and information about the string are in // global variables basesData, String_Start, String_Info, .... static void Put_String_In_Hash(uint32 curID, uint32 i) { String_Ref_t ref = 0; int skip_ct; uint64 key; uint64 key_is_bad; int j; uint32 kmers_skipped = 0; uint32 kmers_bad = 0; uint32 kmers_inserted = 0; char *p = basesData + String_Start[i]; char *window = basesData + String_Start[i]; key = key_is_bad = 0; for (uint32 j=0; j MAX_STRING_NUM) fprintf (stderr, "Too many strings for hash table--exiting\n"), exit(1); setStringRefOffset(ref, TRUELY_ZERO); skip_ct = 0; setStringRefEmpty(ref, TRUELY_ZERO); if (key_is_bad == false) { Hash_Insert(ref, key, window); kmers_inserted++; } else { kmers_bad++; } while (*p != 0) { window++; String_Ref_t newoff = getStringRefOffset(ref) + 1; assert(newoff < OFFSET_MASK); setStringRefOffset(ref, newoff); if (++skip_ct > HASH_KMER_SKIP) skip_ct = 0; key_is_bad >>= 1; key_is_bad |= (uint64) (Char_Is_Bad[(int) * p]) << (G.Kmer_Len - 1); key >>= 2; key |= (uint64) (Bit_Equivalent[(int) * (p ++)]) << (2 * (G.Kmer_Len - 1)); if (skip_ct > 0) { kmers_skipped++; continue; } if (key_is_bad) { kmers_bad++; continue; } Hash_Insert(ref, key, window); kmers_inserted++; } //fprintf(stderr, "STRING %u skipped %u bad %u inserted %u\n", // curID, kmers_skipped, kmers_bad, kmers_inserted); } // Read the next batch of strings from stream and create a hash // table index of their G.Kmer_Len -mers. Return 1 if successful; // 0 otherwise. The batch ends when either end-of-file is encountered // or Max_Hash_Strings have been read in. first_frag_id is the // internal ID of the first fragment in the hash table. int Build_Hash_Index(gkStore *gkpStore, uint32 bgnID, uint32 endID) { String_Ref_t ref; uint64 total_len; uint64 hash_entry_limit; fprintf(stderr, "Build_Hash_Index from "F_U32" to "F_U32"\n", bgnID, endID); Hash_String_Num_Offset = bgnID; String_Ct = 0; Extra_String_Ct = 0; Extra_String_Subcount = MAX_EXTRA_SUBCOUNT; total_len = 0; //if (Data == NULL) { // Extra_Data_Len = Max_Hash_Data_Len + AS_MAX_READLEN; // Data_Len = Max_Hash_Data_Len + AS_MAX_READLEN; // // basesData = new char [Data_Len]; // qualsData = new char [Data_Len]; // // old_ref_len = Data_Len / (HASH_KMER_SKIP + 1); // nextRef = new String_Ref_t [old_ref_len]; //} //memset(nextRef, 0xff, old_ref_len * sizeof(String_Ref_t)); memset(Hash_Table, 0x00, HASH_TABLE_SIZE * sizeof(Hash_Bucket_t)); memset(Hash_Check_Array, 0x00, HASH_TABLE_SIZE * sizeof(Check_Vector_t)); Extra_Ref_Ct = 0; Hash_Entries = 0; hash_entry_limit = G.Max_Hash_Load * HASH_TABLE_SIZE * ENTRIES_PER_BUCKET; #if 0 fprintf(stderr, "HASH LOADING STARTED: fragID %12"F_U64P"\n", first_frag_id); fprintf(stderr, "HASH LOADING STARTED: strings %12"F_U64P" out of %12"F_U64P" max.\n", String_Ct, G.Max_Hash_Strings); fprintf(stderr, "HASH LOADING STARTED: length %12"F_U64P" out of %12"F_U64P" max.\n", total_len, G.Max_Hash_Data_Len); fprintf(stderr, "HASH LOADING STARTED: entries %12"F_U64P" out of %12"F_U64P" max (load %.2f).\n", Hash_Entries, hash_entry_limit, (100.0 * Hash_Entries) / (HASH_TABLE_SIZE * ENTRIES_PER_BUCKET)); #endif // Compute an upper limit on the number of bases we will load. The number of Hash_Entries // can't be computed here, so the real loop below could end earlier than expected - and we // don't use a little bit of memory. uint32 nSkipped = 0; uint32 nShort = 0; uint32 nLoadable = 0; uint64 maxAlloc = 0; uint32 curID = 0; // The last ID loaded into the hash for (curID=bgnID; ((String_Ct < G.Max_Hash_Strings) && (total_len < G.Max_Hash_Data_Len) && (curID <= endID)); curID++) { gkRead *read = gkpStore->gkStore_getRead(curID); if ((read->gkRead_libraryID() < G.minLibToHash) || (read->gkRead_libraryID() > G.maxLibToHash)) { nSkipped++; continue; } if (read->gkRead_sequenceLength() < G.Min_Olap_Len) { nShort++; continue; } nLoadable++; maxAlloc += read->gkRead_sequenceLength() + 1; } fprintf(stderr, "Found "F_U32" reads with length "F_U64" to load; "F_U32" skipped by being too short; "F_U32" skipped per library restriction\n", nLoadable, maxAlloc, nShort, nSkipped); // This should be less than what the user requested on the command line if (maxAlloc >= G.Max_Hash_Data_Len + AS_MAX_READLEN) fprintf(stderr, "maxAlloc = "F_U64" G.Max_Hash_Data_Len = "F_U64" AS_MAX_READLEN = %u\n", maxAlloc, G.Max_Hash_Data_Len, AS_MAX_READLEN); assert(maxAlloc < G.Max_Hash_Data_Len + AS_MAX_READLEN); // Allocate space, then fill it. uint64 nextRef_Len = maxAlloc / (HASH_KMER_SKIP + 1); Extra_Data_Len = Data_Len = maxAlloc; basesData = new char [Data_Len]; qualsData = new char [Data_Len]; nextRef = new String_Ref_t [nextRef_Len]; memset(nextRef, 0xff, sizeof(String_Ref_t) * nextRef_Len); gkReadData *readData = new gkReadData; for (curID=bgnID; ((String_Ct < G.Max_Hash_Strings) && (total_len < G.Max_Hash_Data_Len) && (Hash_Entries < hash_entry_limit) && (curID <= endID)); curID++, String_Ct++) { // Load sequence if it exists, otherwise, add an empty read. // Duplicated in Process_Overlaps(). String_Start[String_Ct] = UINT64_MAX; String_Info[String_Ct].length = 0; String_Info[String_Ct].lfrag_end_screened = TRUE; String_Info[String_Ct].rfrag_end_screened = TRUE; gkRead *read = gkpStore->gkStore_getRead(curID); if ((read->gkRead_libraryID() < G.minLibToHash) || (read->gkRead_libraryID() > G.maxLibToHash)) continue; uint32 len = read->gkRead_sequenceLength(); if (len < G.Min_Olap_Len) continue; gkpStore->gkStore_loadReadData(read, readData); char *seqptr = readData->gkReadData_getSequence(); char *qltptr = readData->gkReadData_getQualities(); // Note where we are going to store the string, and how long it is String_Start[String_Ct] = total_len; String_Info[String_Ct].length = len; String_Info[String_Ct].lfrag_end_screened = FALSE; String_Info[String_Ct].rfrag_end_screened = FALSE; // Store it. for (uint32 i=0; i 0) { uint32 extra = new_len % (HASH_KMER_SKIP + 1); if (extra > 0) new_len += 1 + HASH_KMER_SKIP - extra; } #endif // Trouble - allocate more space for sequence and quality data. // This was computed ahead of time! if (total_len > maxAlloc) fprintf(stderr, "total_len="F_U64" len="F_U32" maxAlloc="F_U64"\n", total_len, len, maxAlloc); assert(total_len <= maxAlloc); // What is Extra_Data_Len? It's set to Data_Len if we would have reallocated here. Put_String_In_Hash(curID, String_Ct); if ((String_Ct % 100000) == 0) fprintf (stderr, "String_Ct:%12"F_U64P"/%12"F_U32P" totalLen:%12"F_U64P"/%12"F_U64P" Hash_Entries:%12"F_U64P"/%12"F_U64P" Load: %.2f%%\n", String_Ct, G.Max_Hash_Strings, total_len, G.Max_Hash_Data_Len, Hash_Entries, hash_entry_limit, 100.0 * Hash_Entries / (HASH_TABLE_SIZE * ENTRIES_PER_BUCKET)); } curID--; // We always stop on the read after we loaded. delete readData; fprintf(stderr, "HASH LOADING STOPPED: strings %12"F_U64P" out of %12"F_U32P" max.\n", String_Ct, G.Max_Hash_Strings); fprintf(stderr, "HASH LOADING STOPPED: length %12"F_U64P" out of %12"F_U64P" max.\n", total_len, G.Max_Hash_Data_Len); fprintf(stderr, "HASH LOADING STOPPED: entries %12"F_U64P" out of %12"F_U64P" max (load %.2f).\n", Hash_Entries, hash_entry_limit, 100.0 * Hash_Entries / (HASH_TABLE_SIZE * ENTRIES_PER_BUCKET)); if (String_Ct == 0) { fprintf(stderr, "HASH LOADING STOPPED: no strings added?\n"); return(endID); } Used_Data_Len = total_len; //fprintf(stderr, "Extra_Ref_Ct = "F_U64" Max_Extra_Ref_Space = "F_U64"\n", Extra_Ref_Ct, Max_Extra_Ref_Space); if (Extra_Ref_Ct > Max_Extra_Ref_Space) { int32 newSize = (Max_Extra_Ref_Space == 0) ? 16 * 1024 : Max_Extra_Ref_Space * 2; while (newSize < Extra_Ref_Ct) newSize *= 2; String_Ref_t *newSpace = new String_Ref_t [newSize]; memcpy(newSpace, Extra_Ref_Space, sizeof(String_Ref_t) * Max_Extra_Ref_Space); delete [] Extra_Ref_Space; Max_Extra_Ref_Space = newSize; // Former max_extra_ref_ct Extra_Ref_Space = newSpace; } if (G.Kmer_Skip_File != NULL) Mark_Skip_Kmers(); // Coalesce reference chain into adjacent entries in Extra_Ref_Space Extra_Ref_Ct = 0; for (int32 i = 0; i < HASH_TABLE_SIZE; i ++) for (int32 j = 0; j < Hash_Table[i].Entry_Ct; j ++) { ref = Hash_Table[i].Entry[j]; if (! getStringRefLast(ref) && ! getStringRefEmpty(ref)) { Extra_Ref_Space[Extra_Ref_Ct] = ref; setStringRefStringNum(Hash_Table[i].Entry[j], (String_Ref_t)(Extra_Ref_Ct >> OFFSET_BITS)); setStringRefOffset (Hash_Table[i].Entry[j], (String_Ref_t)(Extra_Ref_Ct & OFFSET_MASK)); Extra_Ref_Ct ++; do { ref = nextRef[(String_Start[getStringRefStringNum(ref)] + getStringRefOffset(ref)) / (HASH_KMER_SKIP + 1)]; Extra_Ref_Space[Extra_Ref_Ct ++] = ref; } while (! getStringRefLast(ref)); } } return(curID); } canu-1.3/src/overlapInCore/overlapInCore-Find_Overlaps.C000066400000000000000000000276431272335422300232660ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVL/AS_OVL_overlap_common.h * src/AS_OVM/overlapInCore-Find_Overlaps.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-JUL-15 to 2007-NOV-20 * are Copyright 2005,2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2006-MAR-27 to 2006-AUG-21 * are Copyright 2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Art Delcher on 2007-FEB-13 * are Copyright 2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2007-AUG-27 to 2009-JAN-16 * are Copyright 2007,2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2011-MAR-08 to 2011-JUN-17 * are Copyright 2011 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz on 2014-DEC-15 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "overlapInCore.H" // Add information for the match in ref to the list // starting at subscript (* start). The matching window begins // offset bytes from the beginning of this string. static void Add_Match(String_Ref_t ref, int * start, int offset, int * consistent, Work_Area_t * WA) { int * p, save; int diag = 0, new_diag, expected_start = 0, num_checked = 0; int move_to_front = FALSE; new_diag = getStringRefOffset(ref) - offset; for (p = start; (* p) != 0; p = & (WA->Match_Node_Space [(* p)].Next)) { expected_start = WA->Match_Node_Space [(* p)].Start + WA->Match_Node_Space [(* p)].Len - G.Kmer_Len + 1 + HASH_KMER_SKIP; diag = WA->Match_Node_Space [(* p)].Offset - WA->Match_Node_Space [(* p)].Start; if (expected_start < offset) break; if (expected_start == offset) { if (new_diag == diag) { WA->Match_Node_Space [(* p)].Len += 1 + HASH_KMER_SKIP; if (move_to_front) { save = (* p); (* p) = WA->Match_Node_Space [(* p)].Next; WA->Match_Node_Space [save].Next = (* start); (* start) = save; } return; } else move_to_front = TRUE; } num_checked ++; } if (WA->Next_Avail_Match_Node == WA->Match_Node_Size) { int32 newSize = WA->Match_Node_Size * 2; Match_Node_t *newSpace = new Match_Node_t [newSize]; memcpy(newSpace, WA->Match_Node_Space, sizeof(Match_Node_t) * WA->Match_Node_Size); delete [] WA->Match_Node_Space; WA->Match_Node_Size = newSize; WA->Match_Node_Space = newSpace; } if ((* start) != 0 && (num_checked > 0 || abs (diag - new_diag) > 3 || offset < expected_start + G.Kmer_Len - 2)) (* consistent) = FALSE; save = (* start); (* start) = WA->Next_Avail_Match_Node; WA->Next_Avail_Match_Node ++; WA->Match_Node_Space [(* start)].Offset = getStringRefOffset(ref); WA->Match_Node_Space [(* start)].Len = G.Kmer_Len; WA->Match_Node_Space [(* start)].Start = offset; WA->Match_Node_Space [(* start)].Next = save; #if 0 fprintf(stderr, "Add_Match()-- %3d offset %d len %d start %d next %d\n", *start, WA->Match_Node_Space [(* start)].Offset, WA->Match_Node_Space [(* start)].Len, WA->Match_Node_Space [(* start)].Start, WA->Match_Node_Space [(* start)].Next); #endif } // Add information for Ref and all its matches to the global hash table in String_Olap_Space. Grow // the space if necessary. The matching window begins Offset bytes from the beginning of this // string. static void Add_Ref(String_Ref_t Ref, int Offset, Work_Area_t * WA) { uint32 Prev, StrNum, Sub; int consistent; StrNum = getStringRefStringNum(Ref); Sub = (StrNum ^ (StrNum >> STRING_OLAP_SHIFT)) & STRING_OLAP_MASK; while (WA->String_Olap_Space [Sub].Full && WA->String_Olap_Space [Sub].String_Num != StrNum) { Prev = Sub; Sub = WA->String_Olap_Space [Sub].Next; if (Sub == 0) { if (WA->Next_Avail_String_Olap == WA->String_Olap_Size) { int32 newSize = WA->String_Olap_Size * 2; String_Olap_t *newSpace = new String_Olap_t [newSize]; memcpy(newSpace, WA->String_Olap_Space, sizeof(String_Olap_t) * WA->String_Olap_Size); delete [] WA->String_Olap_Space; WA->String_Olap_Size = newSize; WA->String_Olap_Space = newSpace; } Sub = WA->Next_Avail_String_Olap ++; WA->String_Olap_Space [Prev].Next = Sub; WA->String_Olap_Space [Sub].Full = FALSE; break; } } if (! WA->String_Olap_Space [Sub].Full) { WA->String_Olap_Space [Sub].String_Num = StrNum; WA->String_Olap_Space [Sub].Match_List = 0; WA->String_Olap_Space [Sub].diag_sum = 0.0; WA->String_Olap_Space [Sub].diag_ct = 0; WA->String_Olap_Space [Sub].Next = 0; WA->String_Olap_Space [Sub].Full = TRUE; WA->String_Olap_Space [Sub].consistent = TRUE; } consistent = WA->String_Olap_Space [Sub].consistent; WA->String_Olap_Space [Sub].diag_sum += getStringRefOffset(Ref) - Offset; WA->String_Olap_Space [Sub].diag_ct ++; Add_Match (Ref, & (WA->String_Olap_Space [Sub].Match_List), Offset, & consistent, WA); WA->String_Olap_Space [Sub].consistent = consistent; return; } // Search for string S with hash key Key in the global // Hash_Table starting at subscript Sub. Return the matching // reference in the hash table if there is one, or else a reference // with the Empty bit set true. Set (* Where) to the subscript in // Extra_Ref_Space where the reference was found if it was found there. // Set (* hi_hits) to TRUE if hash table entry is found but is empty // because it was screened out, otherwise set to FALSE. static String_Ref_t Hash_Find(uint64 Key, int64 Sub, char * S, int64 * Where, int * hi_hits) { String_Ref_t H_Ref = 0; char * T; unsigned char Key_Check; int64 Ct, Probe; int i; Key_Check = KEY_CHECK_FUNCTION (Key); Probe = PROBE_FUNCTION (Key); (* hi_hits) = FALSE; Ct = 0; do { for (i = 0; i < Hash_Table [Sub].Entry_Ct; i ++) if (Hash_Table [Sub].Check [i] == Key_Check) { int is_empty; H_Ref = Hash_Table [Sub].Entry [i]; //fprintf(stderr, "Href = Hash_Table %u Entry %u = "F_U64"\n", Sub, i, H_Ref); is_empty = getStringRefEmpty(H_Ref); if (! getStringRefLast(H_Ref) && ! is_empty) { (* Where) = ((uint64)getStringRefStringNum(H_Ref) << OFFSET_BITS) + getStringRefOffset(H_Ref); H_Ref = Extra_Ref_Space [(* Where)]; //fprintf(stderr, "Href = Extra_Ref_Space "F_U64" = "F_U64"\n", *Where, H_Ref); } //fprintf(stderr, "Href = "F_U64" Get String_Start[ "F_U64" ] + "F_U64"\n", getStringRefStringNum(H_Ref), getStringRefOffset(H_Ref)); T = basesData + String_Start [getStringRefStringNum(H_Ref)] + getStringRefOffset(H_Ref); if (strncmp (S, T, G.Kmer_Len) == 0) { if (is_empty) { setStringRefEmpty(H_Ref, TRUELY_ONE); (* hi_hits) = TRUE; } return H_Ref; } } if (Hash_Table [Sub].Entry_Ct < ENTRIES_PER_BUCKET) { setStringRefEmpty(H_Ref, TRUELY_ONE); return H_Ref; } Sub = (Sub + Probe) % HASH_TABLE_SIZE; } while (++ Ct < HASH_TABLE_SIZE); setStringRefEmpty(H_Ref, TRUELY_ONE); return H_Ref; } // Find and output all overlaps and branch points between string // Frag and any fragment currently in the global hash table. // Frag_Len is the length of Frag and Frag_Num is its ID number. // Dir is the orientation of Frag . void Find_Overlaps(char Frag [], int Frag_Len, char quality [], uint32 Frag_Num, Direction_t Dir, Work_Area_t * WA) { String_Ref_t Ref; char * P, * Window; uint64 Key, Next_Key; int64 Sub, Next_Sub, Where; Check_Vector_t This_Check, Next_Check; int Offset, Shift, Next_Shift; int hi_hits; int j; memset (WA->String_Olap_Space, 0, STRING_OLAP_MODULUS * sizeof (String_Olap_t)); WA->Next_Avail_String_Olap = STRING_OLAP_MODULUS; WA->Next_Avail_Match_Node = 1; assert (Frag_Len >= G.Kmer_Len); Offset = 0; P = Window = Frag; WA->left_end_screened = FALSE; WA->right_end_screened = FALSE; WA->A_Olaps_For_Frag = 0; WA->B_Olaps_For_Frag = 0; Key = 0; for (j = 0; j < G.Kmer_Len; j ++) Key |= (uint64) (Bit_Equivalent [(int) * (P ++)]) << (2 * j); Sub = HASH_FUNCTION (Key); Shift = HASH_CHECK_FUNCTION (Key); Next_Key = (Key >> 2); Next_Key |= ((uint64) (Bit_Equivalent [(int) * P])) << (2 * (G.Kmer_Len - 1)); Next_Sub = HASH_FUNCTION (Next_Key); Next_Shift = HASH_CHECK_FUNCTION (Next_Key); Next_Check = Hash_Check_Array [Next_Sub]; if ((Hash_Check_Array [Sub] & (((Check_Vector_t) 1) << Shift)) != 0) { Ref = Hash_Find (Key, Sub, Window, & Where, & hi_hits); if (hi_hits) { WA->left_end_screened = TRUE; } if (! getStringRefEmpty(Ref)) { while (TRUE) { if (Frag_Num < getStringRefStringNum(Ref) + Hash_String_Num_Offset) Add_Ref (Ref, Offset, WA); if (getStringRefLast(Ref)) break; else { Ref = Extra_Ref_Space [++ Where]; assert (! getStringRefEmpty(Ref)); } } } } while ((* P) != '\0') { Window ++; Offset ++; Key = Next_Key; Shift = Next_Shift; Sub = Next_Sub; This_Check = Next_Check; P ++; Next_Key = (Key >> 2); Next_Key |= ((uint64) (Bit_Equivalent [(int) * P])) << (2 * (G.Kmer_Len - 1)); Next_Sub = HASH_FUNCTION (Next_Key); Next_Shift = HASH_CHECK_FUNCTION (Next_Key); Next_Check = Hash_Check_Array [Next_Sub]; if ((This_Check & (((Check_Vector_t) 1) << Shift)) != 0) { Ref = Hash_Find (Key, Sub, Window, & Where, & hi_hits); if (hi_hits) { if (Offset < HOPELESS_MATCH) { WA->left_end_screened = TRUE; } if (Frag_Len - Offset - G.Kmer_Len + 1 < HOPELESS_MATCH) { WA->right_end_screened = TRUE; } } if (! getStringRefEmpty(Ref)) { while (TRUE) { if (Frag_Num < getStringRefStringNum(Ref) + Hash_String_Num_Offset) Add_Ref (Ref, Offset, WA); if (getStringRefLast(Ref)) break; else { Ref = Extra_Ref_Space [++ Where]; assert (! getStringRefEmpty(Ref)); } } } } } Process_String_Olaps (Frag, Frag_Len, quality, Frag_Num, Dir, WA); } canu-1.3/src/overlapInCore/overlapInCore-Output.C000066400000000000000000000214341272335422300220230ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVL/AS_OVL_overlap_common.h * src/AS_OVM/overlapInCore-Output.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-JUL-15 to 2007-NOV-20 * are Copyright 2005,2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2006-MAR-27 to 2006-AUG-21 * are Copyright 2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Art Delcher on 2007-FEB-13 * are Copyright 2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2007-AUG-27 to 2009-JAN-16 * are Copyright 2007,2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2011-MAR-08 to 2011-JUN-17 * are Copyright 2011 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz from 2014-NOV-17 to 2015-JUN-16 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-26 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "overlapInCore.H" #include // Output the overlap between strings S_ID and T_ID which // have lengths S_Len and T_Len , respectively. // The overlap information is in (* olap) . // S_Dir indicates the orientation of S . // T is always forward. void Output_Overlap(uint32 S_ID, int S_Len, Direction_t S_Dir, uint32 T_ID, int T_Len, Olap_Info_t *olap, Work_Area_t *WA) { ovOverlap *ovs = WA->overlaps + WA->overlapsLen++; // Overlap is good for UTG only. ovs->dat.ovl.forUTG = true; ovs->dat.ovl.forOBT = false; ovs->dat.ovl.forDUP = false; // Set the span of the alignment. ovs->dat.ovl.span = (olap->s_hi - olap->s_lo); ovs->dat.ovl.span += (olap->t_hi - olap->t_lo); ovs->dat.ovl.span += (olap->delta_ct); assert((ovs->dat.ovl.span % 2) == 0); ovs->dat.ovl.span /= 2; assert (S_ID < T_ID); int32 S_Right_Hang = S_Len - olap->s_hi - 1; int32 T_Right_Hang = T_Len - olap->t_hi - 1; bool Sleft; char orient = 0; int32 ahg = 0; int32 bhg = 0; //fprintf(stderr, "S %d %d = %d len %d T %d %d = %d len %d delta_ct %d delta %d\n", // olap->s_lo, olap->s_hi, olap->s_hi - olap->s_lo, S_Len, // olap->t_lo, olap->t_hi, olap->t_hi - olap->t_lo, T_Len, // olap->delta_ct, olap->delta[0]); assert(olap->s_lo < olap->s_hi); assert(olap->t_lo < olap->t_hi); // Ensure this is a dovetail or contained overlap. assert((olap->s_lo == 0) || (olap->t_lo == 0)); assert((olap->s_hi == S_Len - 1) || (olap->t_hi == T_Len - 1)); if ((olap->s_lo > olap->t_lo) || ((olap->s_lo == olap->t_lo) && (S_Right_Hang > T_Right_Hang))) { assert(olap->t_lo == 0); assert((olap->s_hi == S_Len - 1) || (olap->t_hi == T_Len - 1)); Sleft = true; } else { assert(olap->s_lo == 0); assert((olap->s_hi == S_Len - 1) || (olap->t_hi == T_Len - 1)); Sleft = false; } if (Sleft) { ovs->a_iid = S_ID; ovs->b_iid = T_ID; } else { ovs->a_iid = T_ID; ovs->b_iid = S_ID; } //if (Sleft) { // if (S_Right_Hang >= T_Right_Hang) // overlap_type = AS_CONTAINMENT; // else // overlap_type = AS_DOVETAIL; //} else { // if (T_Right_Hang >= S_Right_Hang) // overlap_type = AS_CONTAINMENT; // else // overlap_type = AS_DOVETAIL; //} if (Sleft) { orient = (S_Dir == FORWARD) ? 'N' : 'O'; ahg = olap->s_lo; bhg = T_Right_Hang - S_Right_Hang; } else { orient = (S_Dir == FORWARD) ? 'N' : 'I'; ahg = olap->t_lo; bhg = S_Right_Hang - T_Right_Hang; } // CMM: Regularize the reverse orientated containment overlaps to a common orientation. // // This catches the case where a reverse orient S (T is always fowrard) is placed // in the A position; we flip the overlap to make S be forward and T be reverse. // if ((orient == 'O') && (S_Right_Hang >= T_Right_Hang)) { orient = 'I'; ahg = -(T_Right_Hang - S_Right_Hang); bhg = -(olap->s_lo); } ovs->erate(olap->quality); switch (orient) { case 'N': ovs->a_hang(ahg); ovs->b_hang(bhg); ovs->dat.ovl.flipped = false; break; case 'I': ovs->a_hang(ahg); ovs->b_hang(bhg); ovs->dat.ovl.flipped = true; break; case 'O': ovs->a_hang(-bhg); ovs->b_hang(-ahg); ovs->dat.ovl.flipped = true; break; case 'A': ovs->a_hang(-bhg); ovs->b_hang(-ahg); ovs->dat.ovl.flipped = false; break; } #if OUTPUT_OVERLAP_DELTAS signed char deltas[2 * AS_READ_MAX_NORMAL_LEN]; signed char *deltaCursor = deltas; if (Sleft == false) for (i = 0; i < olap->delta_ct; i ++) olap->delta [i] *= -1; for (int i = 0; i < olap->delta_ct; i ++) { for (int j = abs (olap->delta [i]); j > 0; j -= AS_LONGEST_DELTA) { if (j > AS_LONGEST_DELTA) *deltaCursor++ = AS_LONG_DELTA_CODE; else *deltaCursor++ = j * Sign (olap->delta [i]); } } *deltaCursor = AS_ENDOF_DELTA_CODE; #endif WA->Total_Overlaps ++; if (bhg <= 0) WA->Contained_Overlap_Ct ++; else WA->Dovetail_Overlap_Ct ++; // We also flush the file at the end of a thread if (WA->overlapsLen >= WA->overlapsMax) { pthread_mutex_lock (& Write_Proto_Mutex); for (int32 zz=0; zzoverlapsLen; zz++) Out_BOF->writeOverlap(WA->overlaps + zz); WA->overlapsLen = 0; pthread_mutex_unlock (& Write_Proto_Mutex); } } void Output_Partial_Overlap(uint32 s_id, uint32 t_id, Direction_t dir, const Olap_Info_t *olap, int s_len, int t_len, Work_Area_t *WA) { Total_Overlaps++; ovOverlap *ovl = WA->overlaps + WA->overlapsLen++; assert(s_id < t_id); ovl->a_iid = s_id; ovl->b_iid = t_id; // Overlap is good for OBT or DUP. It will be refined more during the store build. ovl->dat.ovl.forUTG = false; ovl->dat.ovl.forOBT = true; ovl->dat.ovl.forDUP = true; // Set the span of the alignment. ovl->dat.ovl.span = (olap->s_hi - olap->s_lo); ovl->dat.ovl.span += (olap->t_hi - olap->t_lo); ovl->dat.ovl.span += (olap->delta_ct); assert((ovl->dat.ovl.span % 2) == 0); ovl->dat.ovl.span /= 2; // Convert to canonical form with s forward and use space-based // coordinates if (dir == FORWARD) { ovl->dat.ovl.ahg5 = (olap->s_lo); ovl->dat.ovl.ahg3 = s_len - (olap->s_hi + 1); ovl->dat.ovl.bhg5 = (olap->t_lo); ovl->dat.ovl.bhg3 = t_len - (olap->t_hi + 1); ovl->dat.ovl.flipped = false; //assert(a < b); //assert(c < d); } else { ovl->dat.ovl.ahg5 = s_len - (olap->s_hi + 1); ovl->dat.ovl.ahg3 = (olap->s_lo); ovl->dat.ovl.bhg5 = t_len - (olap->t_hi + 1); ovl->dat.ovl.bhg3 = (olap->t_lo); ovl->dat.ovl.flipped = true; //assert(a < b); //assert(c > d); // Reverse! } ovl->erate(olap->quality); // We also flush the file at the end of a thread if (WA->overlapsLen >= WA->overlapsMax) { pthread_mutex_lock(&Write_Proto_Mutex); for (int32 zz=0; zzoverlapsLen; zz++) Out_BOF->writeOverlap(WA->overlaps + zz); WA->overlapsLen = 0; pthread_mutex_unlock(&Write_Proto_Mutex); } } canu-1.3/src/overlapInCore/overlapInCore-Process_Overlaps.C000066400000000000000000000132701272335422300240130ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVL/AS_OVL_overlap_common.h * src/AS_OVM/overlapInCore-Process_Overlaps.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-JUL-15 to 2007-NOV-20 * are Copyright 2005,2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2006-MAR-27 to 2006-AUG-21 * are Copyright 2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Art Delcher on 2007-FEB-13 * are Copyright 2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2007-AUG-27 to 2009-JAN-16 * are Copyright 2007,2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2011-MAR-08 to 2015-AUG-21 * are Copyright 2011,2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz from 2014-DEC-15 to 2015-AUG-25 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-23 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "overlapInCore.H" #include "AS_UTL_reverseComplement.H" #include // Find and output all overlaps between strings in store and those in the global hash table. // This is the entry point for each compute thread. void * Process_Overlaps(void *ptr){ Work_Area_t *WA = (Work_Area_t *)ptr; gkReadData *readData = new gkReadData; char *bases = new char [AS_MAX_READLEN + 1]; char *quals = new char [AS_MAX_READLEN + 1]; while (WA->bgnID < G.endRefID) { WA->overlapsLen = 0; WA->Total_Overlaps = 0; WA->Contained_Overlap_Ct = 0; WA->Dovetail_Overlap_Ct = 0; WA->Kmer_Hits_Without_Olap_Ct = 0; WA->Kmer_Hits_With_Olap_Ct = 0; WA->Multi_Overlap_Ct = 0; fprintf(stderr, "Thread %02u processes reads "F_U32"-"F_U32"\n", WA->thread_id, WA->bgnID, WA->endID); for (uint32 fi=WA->bgnID; fi<=WA->endID; fi++) { // Load sequence/quality data // Duplicated in Build_Hash_Index() gkRead *read = WA->gkpStore->gkStore_getRead(fi); if ((read->gkRead_libraryID() < G.minLibToRef) || (read->gkRead_libraryID() > G.maxLibToRef)) continue; uint32 len = read->gkRead_sequenceLength(); if (len < G.Min_Olap_Len) continue; WA->gkpStore->gkStore_loadReadData(read, readData); char *seqptr = readData->gkReadData_getSequence(); char *qltptr = readData->gkReadData_getQualities(); for (uint32 i=0; igkRead_readID(), FORWARD, WA); reverseComplement(bases, quals, len); Find_Overlaps(bases, len, quals, read->gkRead_readID(), REVERSE, WA); } // Write out this block of overlaps, no need to keep them in core! // While we have a mutex, also find the next block of things to process. fprintf(stderr, "Thread %02u writes reads "F_U32"-"F_U32" (%u overlaps %u/%u kmer hits with/without overlap)\n", WA->thread_id, WA->bgnID, WA->endID, WA->overlapsLen, WA->Kmer_Hits_With_Olap_Ct, WA->Kmer_Hits_Without_Olap_Ct); pthread_mutex_lock(& Write_Proto_Mutex); // Flush any remaining overlaps. for (int zz=0; zzoverlapsLen; zz++) Out_BOF->writeOverlap(WA->overlaps + zz); WA->overlapsLen = 0; // Update stats Total_Overlaps += WA->Total_Overlaps; Contained_Overlap_Ct += WA->Contained_Overlap_Ct; Dovetail_Overlap_Ct += WA->Dovetail_Overlap_Ct; Kmer_Hits_Without_Olap_Ct += WA->Kmer_Hits_Without_Olap_Ct; Kmer_Hits_With_Olap_Ct += WA->Kmer_Hits_With_Olap_Ct; Multi_Overlap_Ct += WA->Multi_Overlap_Ct; WA->bgnID = G.curRefID; WA->endID = G.curRefID + G.perThread - 1; if (WA->endID > G.endRefID) WA->endID = G.endRefID; G.curRefID = WA->endID + 1; pthread_mutex_unlock(& Write_Proto_Mutex); } delete readData; delete [] bases; delete [] quals; return(ptr); } canu-1.3/src/overlapInCore/overlapInCore-Process_String_Overlaps.C000066400000000000000000000565421272335422300253520ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVL/AS_OVL_overlap_common.h * src/AS_OVM/overlapInCore-Process_String_Overlaps.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-JUL-15 to 2007-NOV-20 * are Copyright 2005,2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2006-MAR-27 to 2006-AUG-21 * are Copyright 2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Art Delcher on 2007-FEB-13 * are Copyright 2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2007-AUG-27 to 2009-JAN-16 * are Copyright 2007,2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2011-MAR-08 to 2011-JUN-17 * are Copyright 2011 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz from 2014-DEC-15 to 2015-JUL-20 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-MAR-23 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "overlapInCore.H" // Choose the best overlap in olap[0 .. (ct - 1)] . // Mark all others as deleted (by setting deleted[] true for them) // and combine their information in the min/max entries in the // best one. static void Combine_Into_One_Olap(Olap_Info_t olap[], int ct, int deleted[]) { int min_diag, max_diag; int s_left_boundary, s_right_boundary; int t_left_boundary, t_right_boundary; int i, best; best = 0; min_diag = olap[0].min_diag; max_diag = olap[0].max_diag; s_left_boundary = olap[0].s_left_boundary; s_right_boundary = olap[0].s_right_boundary; t_left_boundary = olap[0].t_left_boundary; t_right_boundary = olap[0].t_right_boundary; for (i = 1; i < ct; i ++) { if (olap[i].quality < olap[best].quality) best = i; if (olap[i].min_diag < min_diag) min_diag = olap[i].min_diag; if (olap[i].max_diag > max_diag) max_diag = olap[i].max_diag; if (olap[i].s_left_boundary < s_left_boundary) s_left_boundary = olap[i].s_left_boundary; if (olap[i].s_right_boundary > s_right_boundary) s_right_boundary = olap[i].s_right_boundary; if (olap[i].t_left_boundary < t_left_boundary) t_left_boundary = olap[i].t_left_boundary; if (olap[i].t_right_boundary > t_right_boundary) t_right_boundary = olap[i].t_right_boundary; } olap[best].min_diag = min_diag; olap[best].max_diag = max_diag; olap[best].s_left_boundary = s_left_boundary; olap[best].s_right_boundary = s_right_boundary; olap[best].t_left_boundary = t_left_boundary; olap[best].t_right_boundary = t_right_boundary; for (i = 0; i < ct; i ++) deleted[i] = (i != best); } // Combine overlaps whose overlap regions intersect sufficiently // in p[0 .. (ct - 1)] by marking the poorer quality one // deleted (by setting deleted[] true for it) and combining // its min/max info in the other. Assume all entries in // deleted are 0 initially. static void Merge_Intersecting_Olaps(Olap_Info_t p[], int ct, int deleted[]) { int i, j, lo_diag, hi_diag; for (i = 0; i < ct - 1; i ++) for (j = i + 1; j < ct; j ++) { if (deleted[i] || deleted[j]) continue; lo_diag = p[i].min_diag; hi_diag = p[i].max_diag; if ((lo_diag <= 0 && p[j].min_diag > 0) || (lo_diag > 0 && p[j].min_diag <= 0)) continue; if ((lo_diag >= 0 && p[j]. t_right_boundary - lo_diag - p[j].s_left_boundary >= MIN_INTERSECTION) || (lo_diag <= 0 && p[j]. s_right_boundary + lo_diag - p[j]. t_left_boundary >= MIN_INTERSECTION) || (hi_diag >= 0 && p[j]. t_right_boundary - hi_diag - p[j].s_left_boundary >= MIN_INTERSECTION) || (hi_diag <= 0 && p[j]. s_right_boundary + hi_diag - p[j]. t_left_boundary >= MIN_INTERSECTION)) { Olap_Info_t * discard, * keep; if (p[i].quality < p[j].quality) { keep = p + i; discard = p + j; deleted[j] = TRUE; } else { keep = p + j; discard = p + i; deleted[i] = TRUE; } if (discard->min_diag < keep->min_diag) keep->min_diag = discard->min_diag; if (discard->max_diag > keep->max_diag) keep->max_diag = discard->max_diag; if (discard->s_left_boundary < keep->s_left_boundary) keep->s_left_boundary = discard->s_left_boundary; if (discard->s_right_boundary > keep->s_right_boundary) keep->s_right_boundary = discard->s_right_boundary; if (discard->t_left_boundary < keep->t_left_boundary) keep->t_left_boundary = discard->t_left_boundary; if (discard->t_right_boundary > keep->t_right_boundary) keep->t_right_boundary = discard->t_right_boundary; } } } // Add information for the overlap between strings S and T // at positions s_lo .. s_hi and t_lo .. t_hi , resp., and // with quality qual to the array olap[] which // currently has (* ct) entries. Increment (* ct) if this // is a new, distinct overlap; otherwise, modify an existing // entry if this is just a "slide" of an existing overlap. static void Add_Overlap(int s_lo, int s_hi, int t_lo, int t_hi, double qual, Olap_Info_t * olap, int &ct, Work_Area_t * WA) { // If not partials, combine overlapping overlaps if (G.Doing_Partial_Overlaps == false) { int32 new_diag = t_lo - s_lo; for (int32 i=0; i < ct; i++) { int32 old_diag = olap[i].t_lo - olap[i].s_lo; // If intersecting, just extend the existing saved overlap. if ((new_diag > 0 && old_diag > 0 && olap[i].t_right_boundary - new_diag - olap[i].s_left_boundary >= MIN_INTERSECTION) || (new_diag <= 0 && old_diag <= 0 && olap[i].s_right_boundary + new_diag - olap[i].t_left_boundary >= MIN_INTERSECTION)) { if (new_diag < olap[i].min_diag) olap[i].min_diag = new_diag; if (new_diag > olap[i].max_diag) olap[i].max_diag = new_diag; if (s_lo < olap[i].s_left_boundary) olap[i].s_left_boundary = s_lo; if (s_hi > olap[i].s_right_boundary) olap[i].s_right_boundary = s_hi; if (t_lo < olap[i].t_left_boundary) olap[i].t_left_boundary = t_lo; if (t_hi > olap[i].t_right_boundary) olap[i].t_right_boundary = t_hi; // If better quality, copy in the new overlap if (qual < olap[i].quality) { olap[i].s_lo = s_lo; olap[i].s_hi = s_hi; olap[i].t_lo = t_lo; olap[i].t_hi = t_hi; olap[i].quality = qual; //memcpy(& (olap[i].delta), WA->editDist->Left_Delta, WA->editDist->Left_Delta_Len * sizeof (int)); memcpy(olap[i].delta, WA->editDist->Left_Delta, WA->editDist->Left_Delta_Len * sizeof(int32)); olap[i].delta_ct = WA->editDist->Left_Delta_Len; } return; } } } if (ct >= MAX_DISTINCT_OLAPS) { // no room for a new entry; this shouldn't happen //fprintf(stderr, "SKIP - no space left.\n"); return; } // Add a new overlap olap[ct].s_lo = olap[ct].s_left_boundary = s_lo; olap[ct].s_hi = olap[ct].s_right_boundary = s_hi; olap[ct].t_lo = olap[ct].t_left_boundary = t_lo; olap[ct].t_hi = olap[ct].t_right_boundary = t_hi; olap[ct].quality = qual; //memcpy(& (olap[ct].delta), WA->editDist->Left_Delta, WA->editDist->Left_Delta_Len * sizeof (int)); memcpy(olap[ct].delta, WA->editDist->Left_Delta, WA->editDist->Left_Delta_Len * sizeof(int32)); olap[ct].delta_ct = WA->editDist->Left_Delta_Len; olap[ct].min_diag = t_lo - s_lo; olap[ct].max_diag = t_lo - s_lo; ct++; } // Return TRUE iff the exact match region beginning at // position start in the first string and offset in // the second string lies along the alignment from // lo .. hi on the first string where the delta-encoding // of the alignment is given by // WA->Left_Delta[0 .. (WA->Left_Delta_Len-1)] . static int Lies_On_Alignment(int start, int offset, int s_lo, int t_lo, Work_Area_t * WA) { int i, diag, new_diag; diag = t_lo - s_lo; new_diag = offset - start; for (i = 0; i < WA->editDist->Left_Delta_Len; i ++) { s_lo += abs (WA->editDist->Left_Delta[i]); if (start < s_lo) return (abs (new_diag - diag) <= SHIFT_SLACK); if (WA->editDist->Left_Delta[i] < 0) diag ++; else { s_lo ++; diag --; } } return (abs (new_diag - diag) <= SHIFT_SLACK); } // Choose the best partial overlap in olap[0 .. (ct - 1)] . // Set the corresponding deleted entry to false for the others. // Best is the greatest number of matching bases. static void Choose_Best_Partial(Olap_Info_t * olap, int ct, int deleted[]) { double matching_bases; int i, best; best = 0; matching_bases = (1.0 - olap[0].quality) * (2 + olap[0].s_hi - olap[0].s_lo + olap[0].t_hi - olap[0].t_lo); // actually twice the number of matching bases but the max will be // the same overlap for (i = 1; i < ct; i ++) { double mb; mb = (1.0 - olap[i].quality) * (2 + olap[i].s_hi - olap[i].s_lo + olap[i].t_hi - olap[i].t_lo); if (matching_bases < mb || (matching_bases == mb && olap[i].quality < olap[best].quality)) best = i; } for (i = 0; i < ct; i ++) deleted[i] = (i != best); } // Return TRUE iff there is any length-( window_len ) subarray // of a[0 .. (n-1)] that sums to threshold or higher. static int Has_Bad_Window(char *a, int n, int window_len, int threshold) { if (n < window_len) return(FALSE); int32 sum = 0; int32 i=0; int32 j=0; for (i=0; i= threshold) return(TRUE); while (i < n) { sum -= a[j++]; sum += a[i++]; if (sum >= threshold) return(TRUE); } return(FALSE); } // Find and report all overlaps and branch points between string S // (with length S_Len and id S_ID ) and string T (with // length & screen info in t_info and id T_ID ) using the exact // matches in the list beginning at subscript (* Start). Dir is // the orientation of S . static void Process_Matches (int * Start, char * S, int S_Len, char * S_quality, uint32 S_ID, Direction_t Dir, char * T, Hash_Frag_Info_t t_info, char * T_quality, uint32 T_ID, Work_Area_t * WA, int consistent) { int P, * Ref; Olap_Info_t *distinct_olap = NULL; Match_Node_t * Longest_Match, * Ptr; Overlap_t Kind_Of_Olap = NONE; double Quality; int Olap_Len; int overlaps_output = 0; int distinct_olap_ct; int Max_Len, S_Lo, S_Hi, T_Lo, T_Hi; int t_len; int Done_S_Left, Done_S_Right; int Errors; Done_S_Left = Done_S_Right = FALSE; t_len = t_info.length; assert ((* Start) != 0); // If a singleton match is hopeless on either side // it needn't be processed if (G.Use_Hopeless_Check && WA->Match_Node_Space[(* Start)].Next == 0 && ! G.Doing_Partial_Overlaps) { int s_head, t_head, s_tail, t_tail; int is_hopeless = FALSE; s_head = WA->Match_Node_Space[(* Start)].Start; t_head = WA->Match_Node_Space[(* Start)].Offset; if (s_head <= t_head) { if (s_head > HOPELESS_MATCH && ! WA->left_end_screened) is_hopeless = TRUE; } else { if (t_head > HOPELESS_MATCH && ! t_info.lfrag_end_screened) is_hopeless = TRUE; } s_tail = S_Len - s_head - WA->Match_Node_Space[(* Start)].Len + 1; t_tail = t_len - t_head - WA->Match_Node_Space[(* Start)].Len + 1; if (s_tail <= t_tail) { if (s_tail > HOPELESS_MATCH && ! WA->right_end_screened) is_hopeless = TRUE; } else { if (t_tail > HOPELESS_MATCH && ! t_info.rfrag_end_screened) is_hopeless = TRUE; } if (is_hopeless) { (* Start) = 0; WA->Kmer_Hits_Without_Olap_Ct ++; return; } } distinct_olap = WA->distinct_olap; distinct_olap_ct = 0; while ((* Start) != 0) { int a_hang, b_hang; int hit_limit = FALSE; Max_Len = WA->Match_Node_Space[(* Start)].Len; Longest_Match = WA->Match_Node_Space + (* Start); for (P = WA->Match_Node_Space[(* Start)].Next; P != 0; P = WA->Match_Node_Space[P].Next) if (WA->Match_Node_Space[P].Len > Max_Len) { Max_Len = WA->Match_Node_Space[P].Len; Longest_Match = WA->Match_Node_Space + P; } a_hang = Longest_Match->Start - Longest_Match->Offset; b_hang = a_hang + S_Len - t_len; hit_limit = ((WA->A_Olaps_For_Frag >= G.Frag_Olap_Limit && a_hang <= 0) || (WA->B_Olaps_For_Frag >= G.Frag_Olap_Limit && b_hang <= 0)); if (! hit_limit) { //fprintf(stderr, "Extend_Alignment()- start %d len %d offset %d diag %d - S ID %u %d-%d - T ID %u %d-%d\n", // Longest_Match->Start, // Longest_Match->Len, // Longest_Match->Offset, // Longest_Match->Start - Longest_Match->Offset, // S_ID, S_Lo, S_Hi, T_ID, T_Lo, T_Hi); Kind_Of_Olap = WA->editDist->Extend_Alignment(Longest_Match, S, S_Len, T, t_len, S_Lo, S_Hi, T_Lo, T_Hi, Errors); if (Kind_Of_Olap == DOVETAIL || G.Doing_Partial_Overlaps) { if (1 + S_Hi - S_Lo >= G.Min_Olap_Len && 1 + T_Hi - T_Lo >= G.Min_Olap_Len) { Olap_Len = 1 + MIN (S_Hi - S_Lo, T_Hi - T_Lo); Quality = (double) Errors / Olap_Len; if (Errors <= WA->editDist->Error_Bound[Olap_Len]) { //fprintf(stderr, "Add_Overlap()- quality %f count %d\n", Quality, distinct_olap_ct); Add_Overlap (S_Lo, S_Hi, T_Lo, T_Hi, Quality, distinct_olap, distinct_olap_ct, WA); } } } } if (consistent) (* Start) = 0; for (Ref = Start; (* Ref) != 0; ) { Ptr = WA->Match_Node_Space + (* Ref); if (Ptr == Longest_Match || ((Kind_Of_Olap == DOVETAIL || G.Doing_Partial_Overlaps) && S_Lo - SHIFT_SLACK <= Ptr->Start && Ptr->Start + Ptr->Len <= (S_Hi + 1) + SHIFT_SLACK - 1 && Lies_On_Alignment (Ptr->Start, Ptr->Offset, S_Lo, T_Lo, WA) )) (* Ref) = Ptr->Next; // Remove this node, it matches the alignment else Ref = & (Ptr->Next); } } if (distinct_olap_ct > 0) { int deleted[MAX_DISTINCT_OLAPS] = {0}; Olap_Info_t * p; int i; // Check if any previously distinct overlaps should be merged because // of other merges. if (G.Doing_Partial_Overlaps) { if (G.Unique_Olap_Per_Pair) Choose_Best_Partial (distinct_olap, distinct_olap_ct, deleted); //else // Do nothing, output them all } else { if (G.Unique_Olap_Per_Pair) Combine_Into_One_Olap (distinct_olap, distinct_olap_ct, deleted); else Merge_Intersecting_Olaps (distinct_olap, distinct_olap_ct, deleted); } p = distinct_olap; for (i = 0; i < distinct_olap_ct; i ++) { if (! deleted[i]) { bool rejected = FALSE; if (G.Use_Window_Filter) { int32 d; int32 i = p->s_lo;; int32 j = p->t_lo; int32 q_len = 0; char *q_diff = WA->q_diff; for (int32 k=0; kdelta_ct; k++) { int32 len = abs(p->delta[k]); for (int32 n = 1; n < len; n++) { if (S[i] == T[j] || S[i] == 'n' || T[j] == 'n') { d = 0; } else { d = MIN (S_quality[i], T_quality[j]); d = MIN (d, QUALITY_CUTOFF); } q_diff[q_len++] = d; i++; j++; } if (p->delta[k] > 0) { d = S_quality[i]; i++; } else { d = T_quality[j]; j++; } q_diff[q_len++] = MIN (d, QUALITY_CUTOFF); } while (i <= p->s_hi) { if (S[i] == T[j] || S[i] == 'n' || T[j] == 'n') { d = 0; } else { d = MIN(S_quality[i], T_quality[j]); d = MIN(d, QUALITY_CUTOFF); } q_diff[q_len++] = d; i++; j++; } if (Has_Bad_Window(q_diff, q_len, BAD_WINDOW_LEN, BAD_WINDOW_VALUE)) { rejected = TRUE; Bad_Short_Window_Ct++; } else if (Has_Bad_Window(q_diff, q_len, 100, 240)) { rejected = TRUE; Bad_Long_Window_Ct++; } } if (! rejected) { if (G.Doing_Partial_Overlaps) Output_Partial_Overlap(S_ID, T_ID, Dir, p, S_Len, t_len, WA); else Output_Overlap(S_ID, S_Len, Dir, T_ID, t_len, p, WA); overlaps_output++; if (p->s_lo == 0) WA->A_Olaps_For_Frag++; if (p->s_hi >= S_Len - 1) WA->B_Olaps_For_Frag++; } } p++; } } if (overlaps_output == 0) WA->Kmer_Hits_Without_Olap_Ct++; else { WA->Kmer_Hits_With_Olap_Ct++; if (overlaps_output > 1) WA->Multi_Overlap_Ct++; } return; } // Compare the diag_sum fields in a and b as (String_Olap_t *) 's and // return -1 if a < b , 0 if a == b , and 1 if a > b . // Used for qsort . static int By_Diag_Sum (const void * a, const void * b) { String_Olap_t * x, * y; x = (String_Olap_t *) a; y = (String_Olap_t *) b; if (x->diag_sum < y->diag_sum) return -1; else if (x->diag_sum > y->diag_sum) return 1; return 0; } // Find and report all overlaps and branch points between string S // and all strings in global String_Olap_Space . // Return the number of entries processed. // Len is the length of S , ID is its fragment ID and // Dir indicates if S is forward, or reverse-complemented. int Process_String_Olaps (char * S, int Len, char * S_quality, uint32 ID, Direction_t Dir, Work_Area_t * WA) { int32 i, ct, root_num, start, processed_ct; // Move all full entries to front of String_Olap_Space and set // diag_sum to average diagonal. if enough entries to bother, // sort by average diagonal. Then process positive & negative diagonals // separately in order from longest to shortest overlap. Stop // processing when output limit has been reached. for (i = ct = 0; i < WA->Next_Avail_String_Olap; i ++) if (WA->String_Olap_Space[i].Full) { root_num = WA->String_Olap_Space[i].String_Num; if (root_num + Hash_String_Num_Offset > ID) { if (WA->String_Olap_Space[i].Match_List == 0) { fprintf (stderr, " Curr_String_Num = %d root_num %d have no matches\n", ID, root_num); exit (-2); } if (i != ct) WA->String_Olap_Space[ct] = WA->String_Olap_Space[i]; assert (WA->String_Olap_Space[ct].diag_ct > 0); WA->String_Olap_Space[ct].diag_sum /= WA->String_Olap_Space[ct].diag_ct; ct ++; } } if (ct == 0) return ct; if (ct <= G.Frag_Olap_Limit) { for (i = 0; i < ct; i ++) { root_num = WA->String_Olap_Space[i].String_Num; Process_Matches(&WA->String_Olap_Space[i].Match_List, S, Len, S_quality, ID, Dir, basesData + String_Start[root_num], String_Info[root_num], qualsData + String_Start[root_num], root_num + Hash_String_Num_Offset, WA, WA->String_Olap_Space[i].consistent); assert(WA->String_Olap_Space[i].Match_List == 0); } return ct; } qsort (WA->String_Olap_Space, ct, sizeof (String_Olap_t), By_Diag_Sum); for (start = 0; start < ct && WA->String_Olap_Space[start].diag_sum < 0; start ++) ; processed_ct = 0; for (i = start; i < ct && WA->A_Olaps_For_Frag < G.Frag_Olap_Limit ; i ++) { root_num = WA->String_Olap_Space[i].String_Num; Process_Matches(&WA->String_Olap_Space[i].Match_List, S, Len, S_quality, ID, Dir, basesData + String_Start[root_num], String_Info[root_num], qualsData + String_Start[root_num], root_num + Hash_String_Num_Offset, WA, WA->String_Olap_Space[i].consistent); assert(WA->String_Olap_Space[i].Match_List == 0); processed_ct ++; } for (i = start - 1; i >= 0 && WA->B_Olaps_For_Frag < G.Frag_Olap_Limit ; i --) { root_num = WA->String_Olap_Space[i].String_Num; Process_Matches(&WA->String_Olap_Space[i].Match_List, S, Len, S_quality, ID, Dir, basesData + String_Start[root_num], String_Info[root_num], qualsData + String_Start[root_num], root_num + Hash_String_Num_Offset, WA, WA->String_Olap_Space[i].consistent); assert(WA->String_Olap_Space[i].Match_List == 0); processed_ct ++; } return processed_ct; } canu-1.3/src/overlapInCore/overlapInCore.C000066400000000000000000000525101272335422300205240ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVL/AS_OVL_driver_common.h * src/AS_OVM/overlapInCore.C * * Modifications by: * * Michael Schatz from 2004-SEP-23 to 2012-JAN-26 * are Copyright 2004,2012 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2009,2011-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-AUG-11 to 2015-AUG-25 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-27 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2015-NOV-20 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "overlapInCore.H" #include "AS_UTL_decodeRange.H" oicParameters G; uint32 STRING_NUM_BITS = 31; // MUST BE EXACTLY THIS uint32 OFFSET_BITS = 31; uint64 TRUELY_ONE = (uint64)1; uint64 TRUELY_ZERO = (uint64)0; uint64 STRING_NUM_MASK = (TRUELY_ONE << STRING_NUM_BITS) - 1; uint64 OFFSET_MASK = (TRUELY_ONE << OFFSET_BITS) - 1; uint64 MAX_STRING_NUM = STRING_NUM_MASK; int64 Bad_Short_Window_Ct = 0; // The number of overlaps rejected because of too many errors in a small window int64 Bad_Long_Window_Ct = 0; // The number of overlaps rejected because of too many errors in a long window // Stores sequence and quality data of fragments in hash table char *basesData = NULL; char *qualsData = NULL; size_t Data_Len = 0; String_Ref_t *nextRef = NULL; size_t Extra_Data_Len; // Total length available for hash table string data, // including both regular strings and extra strings // added from kmer screening uint64 Max_Extra_Ref_Space = 0; // allocated amount uint64 Extra_Ref_Ct = 0; // used amount String_Ref_t *Extra_Ref_Space = NULL; uint64 Extra_String_Ct = 0; // Number of extra strings of screen kmers added to hash table uint64 Extra_String_Subcount = 0; // Number of kmers already added to last extra string in hash table Check_Vector_t * Hash_Check_Array = NULL; // Bit vector to eliminate impossible hash matches uint64 Hash_String_Num_Offset = 1; Hash_Bucket_t * Hash_Table; uint64 Kmer_Hits_With_Olap_Ct = 0; uint64 Kmer_Hits_Without_Olap_Ct = 0; uint64 Multi_Overlap_Ct = 0; uint64 String_Ct; // Number of fragments in the hash table Hash_Frag_Info_t * String_Info = NULL; int64 * String_Start = NULL; uint32 String_Start_Size = 0; // Number of available positions in String_Start size_t Used_Data_Len = 0; // Number of bytes of Data currently occupied, including // regular strings and extra kmer screen strings int32 Bit_Equivalent[256] = {0}; // Table to convert characters to 2-bit integer code int32 Char_Is_Bad[256] = {0}; // Table to check if character is not a, c, g or t. uint64 Hash_Entries = 0; uint64 Total_Overlaps = 0; uint64 Contained_Overlap_Ct = 0; uint64 Dovetail_Overlap_Ct = 0; uint64 HSF1 = 666; uint64 HSF2 = 666; uint64 SV1 = 666; uint64 SV2 = 666; uint64 SV3 = 666; pthread_mutex_t Write_Proto_Mutex; ovFile *Out_BOF = NULL; // Allocate memory for (* WA) and set initial values. // Set thread_id field to id . void Initialize_Work_Area(Work_Area_t *WA, int id, gkStore *gkpStore) { uint64 allocated = 0; WA->String_Olap_Size = INIT_STRING_OLAP_SIZE; WA->String_Olap_Space = new String_Olap_t [WA->String_Olap_Size]; WA->Match_Node_Size = INIT_MATCH_NODE_SIZE; WA->Match_Node_Space = new Match_Node_t [WA->Match_Node_Size]; allocated += WA->String_Olap_Size * sizeof (String_Olap_t); allocated += WA->Match_Node_Size * sizeof (Match_Node_t); WA->status = 0; WA->thread_id = id; WA->gkpStore = gkpStore; WA->overlapsLen = 0; WA->overlapsMax = 1024 * 1024 / sizeof(ovOverlap); WA->overlaps = ovOverlap::allocateOverlaps(WA->gkpStore, WA->overlapsMax); allocated += sizeof(ovOverlap) * WA->overlapsMax; fprintf(stderr, "Initialize_Work_Area()-- new prefixEditDistance\n"); WA->editDist = new prefixEditDistance(G.Doing_Partial_Overlaps, G.maxErate); WA->q_diff = new char [AS_MAX_READLEN]; WA->distinct_olap = new Olap_Info_t [MAX_DISTINCT_OLAPS]; fprintf(stderr, "Initialize_Work_Area()-- done\n"); } void Delete_Work_Area(Work_Area_t *WA) { delete WA->editDist; delete [] WA->String_Olap_Space; delete [] WA->Match_Node_Space; delete [] WA->overlaps; delete [] WA->distinct_olap; delete [] WA->q_diff; } int OverlapDriver(void) { fprintf(stderr, "OverlapDriver()--\n"); pthread_t *thread_id = new pthread_t [G.Num_PThreads]; fprintf(stderr, "OverlapDriver()-- WA\n"); Work_Area_t *thread_wa = new Work_Area_t [G.Num_PThreads]; fprintf(stderr, "OverlapDriver()-- gkpStore\n"); gkStore *gkpStore = gkStore::gkStore_open(G.Frag_Store_Path); pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setstacksize(&attr, THREAD_STACKSIZE); pthread_mutex_init(&Write_Proto_Mutex, NULL); for (uint32 i=0; igkStore_getNumReads() < G.endHashID) G.endHashID = gkpStore->gkStore_getNumReads(); // Note distinction between the local bgn/end and the global G.bgn/G.end. uint32 bgnHashID = G.bgnHashID; uint32 endHashID = G.bgnHashID + G.Max_Hash_Strings - 1; // Inclusive! // Iterate over read blocks, build a hash table, then search in threads. //fprintf(stderr, "OverlapDriver()-- Loop top\n"); while (bgnHashID < G.endHashID) { if (endHashID > G.endHashID) endHashID = G.endHashID; assert(0 < bgnHashID); assert(bgnHashID <= endHashID); assert(endHashID <= gkpStore->gkStore_getNumReads()); // Load as much as we can. If we load less than expected, the endHashID is updated to reflect // the last read loaded. //fprintf(stderr, "OverlapDriver()-- Build_Hash_Index\n"); endHashID = Build_Hash_Index(gkpStore, bgnHashID, endHashID); //fprintf(stderr, "Index built.\n"); // Decide the range of reads to process. No more than what is loaded in the table. if (G.bgnRefID < 1) G.bgnRefID = 1; if (G.endRefID > gkpStore->gkStore_getNumReads()) G.endRefID = gkpStore->gkStore_getNumReads(); G.curRefID = G.bgnRefID; // The old version used to further divide the ref range into blocks of at most // Max_Reads_Per_Batch so that those reads could be loaded into core. We don't // need to do that anymore. G.perThread = 1 + (G.endRefID - G.bgnRefID) / G.Num_PThreads / 8; fprintf(stderr, "\n"); fprintf(stderr, "Range: %u-%u. Store has %u reads.\n", G.bgnRefID, G.endRefID, gkpStore->gkStore_getNumReads()); fprintf(stderr, "Chunk: "F_U32" reads/thread -- (G.endRefID="F_U32" - G.bgnRefID="F_U32") / G.Num_PThreads="F_U32" / 8\n", G.perThread, G.endRefID, G.bgnRefID, G.Num_PThreads); fprintf(stderr, "\n"); fprintf(stderr, "Starting "F_U32"-"F_U32" with "F_U32" per thread\n", G.bgnRefID, G.endRefID, G.perThread); fprintf(stderr, "\n"); for (uint32 i=0; i G.endRefID) G.endRefID = G.endRefID; int status = pthread_create(thread_id+i, &attr, Process_Overlaps, thread_wa+i); if (status != 0) fprintf(stderr, "pthread_create error: %s\n", strerror(status)), exit(1); } // The master thread just sits here and waits. for (uint32 i=0; igkStore_close(); for (uint32 i=0; i 0.06) { if (G.Use_Window_Filter) fprintf(stderr, "High error rates requested -- window-filter turned off despite -w flag!\n"); G.Use_Window_Filter = FALSE; G.Use_Hopeless_Check = FALSE; } if (G.Max_Hash_Strings == 0) fprintf(stderr, "* No memory model supplied; -M needed!\n"), err++; if (G.Kmer_Len == 0) fprintf(stderr, "* No kmer length supplied; -k needed!\n"), err++; if (G.Max_Hash_Strings > MAX_STRING_NUM) fprintf(stderr, "Too many strings (--hashstrings), must be less than "F_U64"\n", MAX_STRING_NUM), err++; if (G.Outfile_Name == NULL) fprintf (stderr, "ERROR: No output file name specified\n"), err++; if ((err) || (G.Frag_Store_Path == NULL)) { fprintf(stderr, "USAGE: %s [options] \n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, "-b in contig mode, specify the output file\n"); fprintf(stderr, "-c contig mode. Use 2 frag stores. First is\n"); fprintf(stderr, " for reads; second is for contigs\n"); fprintf(stderr, "-G do partial overlaps\n"); fprintf(stderr, "-h to specify fragments to put in hash table\n"); fprintf(stderr, " Implies LSF mode (no changes to frag store)\n"); fprintf(stderr, "-I designate a file of frag iids to limit olaps to\n"); fprintf(stderr, " (Contig mode only)\n"); fprintf(stderr, "-k if one or two digits, the length of a kmer, otherwise\n"); fprintf(stderr, " the filename containing a list of kmers to ignore in\n"); fprintf(stderr, " the hash table\n"); fprintf(stderr, "-l specify the maximum number of overlaps per\n"); fprintf(stderr, " fragment-end per batch of fragments.\n"); fprintf(stderr, "-m allow multiple overlaps per oriented fragment pair\n"); fprintf(stderr, "-M specify memory size. Valid values are '8GB', '4GB',\n"); fprintf(stderr, " '2GB', '1GB', '256MB'. (Not for Contig mode)\n"); fprintf(stderr, "-o specify output file name\n"); fprintf(stderr, "-P write protoIO output (if not -G)\n"); fprintf(stderr, "-r specify old fragments to overlap\n"); fprintf(stderr, "-t use parallel threads\n"); fprintf(stderr, "-u allow only 1 overlap per oriented fragment pair\n"); fprintf(stderr, "-w filter out overlaps with too many errors in a window\n"); fprintf(stderr, "-z skip the hopeless check\n"); fprintf(stderr, "\n"); fprintf(stderr, "--maxerate only output overlaps with fraction or less error (e.g., 0.06 == 6%%)\n"); fprintf(stderr, "--minlength only output overlaps of or more bases\n"); fprintf(stderr, "\n"); fprintf(stderr, "--hashbits n Use n bits for the hash mask.\n"); fprintf(stderr, "--hashstrings n Load at most n strings into the hash table at one time.\n"); fprintf(stderr, "--hashdatalen n Load at most n bytes into the hash table at one time.\n"); fprintf(stderr, "--hashload f Load to at most 0.0 < f < 1.0 capacity (default 0.7).\n"); fprintf(stderr, "\n"); fprintf(stderr, "--maxreadlen n For batches with all short reads, pack bits differently to\n"); fprintf(stderr, " process more reads per batch.\n"); fprintf(stderr, " all reads must be shorter than n\n"); fprintf(stderr, " --hashstrings limited to 2^(30-m)\n"); fprintf(stderr, " Common values:\n"); fprintf(stderr, " maxreadlen 2048->hashstrings 524288 (default)\n"); fprintf(stderr, " maxreadlen 512->hashstrings 2097152\n"); fprintf(stderr, " maxreadlen 128->hashstrings 8388608\n"); fprintf(stderr, "\n"); fprintf(stderr, "--readsperbatch n Force batch size to n.\n"); fprintf(stderr, "--readsperthread n Force each thread to process n reads.\n"); fprintf(stderr, "\n"); exit(1); } Out_BOF = new ovFile(G.Outfile_Name, ovFileFullWrite); // We know enough now to set the hash function variables, and some other random variables. HSF1 = G.Kmer_Len - (G.Hash_Mask_Bits / 2); HSF2 = 2 * G.Kmer_Len - G.Hash_Mask_Bits; SV1 = HSF1 + 2; SV2 = (HSF1 + HSF2) / 2; SV3 = HSF2 - 2; // Log parameters. fprintf(stderr, "\n"); fprintf(stderr, "STRING_NUM_BITS "F_U32"\n", STRING_NUM_BITS); fprintf(stderr, "OFFSET_BITS "F_U32"\n", OFFSET_BITS); fprintf(stderr, "STRING_NUM_MASK "F_U64"\n", STRING_NUM_MASK); fprintf(stderr, "OFFSET_MASK "F_U64"\n", OFFSET_MASK); fprintf(stderr, "MAX_STRING_NUM "F_U64"\n", MAX_STRING_NUM); fprintf(stderr, "\n"); fprintf(stderr, "Hash_Mask_Bits "F_U32"\n", G.Hash_Mask_Bits); fprintf(stderr, "Max_Hash_Strings "F_U32"\n", G.Max_Hash_Strings); fprintf(stderr, "Max_Hash_Data_Len "F_U64"\n", G.Max_Hash_Data_Len); fprintf(stderr, "Max_Hash_Load %f\n", G.Max_Hash_Load); fprintf(stderr, "Kmer Length "F_U64"\n", G.Kmer_Len); fprintf(stderr, "Min Overlap Length %d\n", G.Min_Olap_Len); fprintf(stderr, "Max Error Rate %f\n", G.maxErate); fprintf(stderr, "\n"); fprintf(stderr, "Num_PThreads "F_U32"\n", G.Num_PThreads); assert (8 * sizeof (uint64) > 2 * G.Kmer_Len); Bit_Equivalent['a'] = Bit_Equivalent['A'] = 0; Bit_Equivalent['c'] = Bit_Equivalent['C'] = 1; Bit_Equivalent['g'] = Bit_Equivalent['G'] = 2; Bit_Equivalent['t'] = Bit_Equivalent['T'] = 3; for (int i = 0; i < 256; i ++) { char ch = tolower ((char) i); if (ch == 'a' || ch == 'c' || ch == 'g' || ch == 't') Char_Is_Bad[i] = 0; else Char_Is_Bad[i] = 1; } fprintf(stderr, "\n"); fprintf(stderr, "HASH_TABLE_SIZE "F_U32"\n", HASH_TABLE_SIZE); fprintf(stderr, "sizeof(Hash_Bucket_t) "F_SIZE_T"\n", sizeof(Hash_Bucket_t)); fprintf(stderr, "hash table size: "F_SIZE_T" MB\n", (HASH_TABLE_SIZE * sizeof(Hash_Bucket_t)) >> 20); fprintf(stderr, "\n"); Hash_Table = new Hash_Bucket_t [HASH_TABLE_SIZE]; fprintf(stderr, "check "F_SIZE_T" MB\n", (HASH_TABLE_SIZE * sizeof (Check_Vector_t) >> 20)); fprintf(stderr, "info "F_SIZE_T" MB\n", (G.Max_Hash_Strings * sizeof (Hash_Frag_Info_t) >> 20)); fprintf(stderr, "start "F_SIZE_T" MB\n", (G.Max_Hash_Strings * sizeof (int64) >> 20)); fprintf(stderr, "\n"); Hash_Check_Array = new Check_Vector_t [HASH_TABLE_SIZE]; String_Info = new Hash_Frag_Info_t [G.Max_Hash_Strings]; String_Start = new int64 [G.Max_Hash_Strings]; String_Start_Size = G.Max_Hash_Strings; memset(Hash_Check_Array, 0, sizeof(Check_Vector_t) * HASH_TABLE_SIZE); memset(String_Info, 0, sizeof(Hash_Frag_Info_t) * G.Max_Hash_Strings); memset(String_Start, 0, sizeof(int64) * G.Max_Hash_Strings); OverlapDriver(); delete [] basesData; delete [] qualsData; delete [] nextRef; delete [] String_Start; delete [] String_Info; delete [] Hash_Check_Array; delete [] Hash_Table; delete Out_BOF; FILE *stats = stderr; if (G.Outstat_Name != NULL) { errno = 0; stats = fopen(G.Outstat_Name, "w"); if (errno) { fprintf(stderr, "WARNING: failed to open '%s' for writing: %s\n", G.Outstat_Name, strerror(errno)); stats = stderr; } } fprintf(stats, " Kmer hits without olaps = "F_S64"\n", Kmer_Hits_Without_Olap_Ct); fprintf(stats, " Kmer hits with olaps = "F_S64"\n", Kmer_Hits_With_Olap_Ct); fprintf(stats, " Multiple overlaps/pair = "F_S64"\n", Multi_Overlap_Ct); fprintf(stats, " Total overlaps produced = "F_S64"\n", Total_Overlaps); fprintf(stats, " Contained overlaps = "F_S64"\n", Contained_Overlap_Ct); fprintf(stats, " Dovetail overlaps = "F_S64"\n", Dovetail_Overlap_Ct); fprintf(stats, "Rejected by short window = "F_S64"\n", Bad_Short_Window_Ct); fprintf(stats, " Rejected by long window = "F_S64"\n", Bad_Long_Window_Ct); if (stats != stderr) fclose(stats); return(0); } canu-1.3/src/overlapInCore/overlapInCore.H000066400000000000000000000414051272335422300205320ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVL/AS_OVL_overlap.h * src/AS_OVM/overlapInCore.H * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005,2007-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2005-SEP-29 to 2006-AUG-23 * are Copyright 2005-2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Art Delcher on 2007-FEB-13 * are Copyright 2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2011-MAR-08 * are Copyright 2011 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz from 2014-DEC-15 to 2015-AUG-25 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-27 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2015-NOV-20 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include "prefixEditDistance.H" #include #ifndef OVERLAPINCORE_H #define OVERLAPINCORE_H #define HASH_KMER_SKIP 0 // Skip this many kmers between the kmers put into the hash // table. Setting this to 0 will make every kmer go // into the hash table. #define BAD_WINDOW_LEN 50 // Length of window in which to look for clustered errors // to invalidate an overlap #define BAD_WINDOW_VALUE (8 * QUALITY_CUTOFF) // This many or more errors in a window of BAD_WINDOW_LEN // invalidates an overlap #define CHECK_MASK 0xff // To set Check field in hash bucket #define DEFAULT_HI_HIT_LIMIT INT_MAX // Any kmer in hash table with at least this many hits // cannot initiate an overlap. Can be changed on the command // line with -K option #define DELETED_FRAG 2 // Indicates fragment was marked as deleted in the store #define DISPLAY_WIDTH 60 // Number of characters per line when displaying sequences #define ENTRIES_PER_BUCKET 21 // In main hash table. Recommended values are 21, 31 or 42 // depending on cache line size. #define HASH_CHECK_MASK 0x1f // Used to set and check bit in Hash_Check_Array // Change if change Check_Vector_t #define HASH_EXPANSION_FACTOR 1.4 // Hash table size is >= this times MAX_HASH_STRINGS #define HASH_MASK ((1 << G.Hash_Mask_Bits) - 1) // Extract right Hash_Mask_Bits bits of hash key #define HASH_TABLE_SIZE (1 + HASH_MASK) // Number of buckets in hash table #define HIGHEST_KMER_LIMIT 255 // If Hi_Hit_Limit is more than this, it's ignored #define HOPELESS_MATCH 90 // A string this long or longer without an exact kmer // match is assumed to be hopeless to find a match // within the error threshold #define IID_GAP_LIMIT 100 // When using a list of fragment IID's, gaps between // IID's this long or longer force a new load partial // store to be done #define INIT_MATCH_NODE_SIZE 10000 // Initial number of nodes to hold exact matches #define INIT_SCREEN_MATCHES 50 // Initial number of screen-match entries per fragment #define INIT_STRING_OLAP_SIZE 5000 // Initial number of different New fragments that // overlap a single Old fragment #define K_MER_STEP 1 // 1 = every k-mer in search // 2 = every other k-mer // 3 = every third, ... // Used to skip some k-mers in finding matches #define MAX_BRANCH_COUNT UCHAR_MAX // The largest branch point count before an overflow #define MAX_DISTINCT_OLAPS 3 // Most possible really different overlaps (i.e., not // just shifts from periodic regions) between 2 fragments // in a given orientation. For fragments of approximately // same size, should never be more than 2. #define EXPECTED_STRING_LEN (AS_MAX_READLEN / 2) #define INITIAL_DATA_LEN (EXPECTED_STRING_LEN * Max_Hash_Strings) // The number of bytes to allocate initially for hash-table sequence #define MAX_LINE_LEN 1000 // Maximum input line when reading FASTA file #define MAX_NAME_LEN 500 // Longest file name allowed #define MIN_CALC_KMER 4 // When calculating the Hi_Hit_Limit based on genome length, etc, // don't set it below this #define MIN_INTERSECTION 10 // Minimum length of match region to be worth reporting #define MIN_OLAP_OUTSIDE_SCREEN 30 // Minimum number of bases outside of screened regions // to be a reportable overlap. Entire overlap (including screened // portion) must still be MIN_OLAP_LEN . #define OUTPUT_OVERLAP_DELTAS 0 // If true include delta-encoding of overlap alignment // in overlap messages. Otherwise, omit them. // As of 6 Oct 2008, support for overlap deltas has been removed. // However, there are enough remnants in AS_MSG to output them. // Just enabling OUTPUT_OVERLAP_DELTAS will not compile; see // AS_MSG_USE_OVL_DELTA in AS_MSG. #define PROBE_MASK 0x3e // Used to determine probe step to resolve collisions #define QUALITY_CUTOFF 20 // Regard quality values higher than this as equal to this // for purposes of finding bad windows #define SCRIPT_NAME "lsf-ovl" // Default name of script produced by make-ovl-script #define SHIFT_SLACK 1 // Allow to be off by this many bases in combining/comparing alignments #define STRING_OLAP_SHIFT 8 // To compute hash function into the String_Olap hash table. #define STRING_OLAP_MODULUS (1 << STRING_OLAP_SHIFT) // The size of the String_Olap hash table. The rest of // the space is used for chaining. This number should be // relatively small to reflect the number of fragments a // given fragment has exact matches with. #define STRING_OLAP_MASK (STRING_OLAP_MODULUS - 1) // To compute hash function into the String_Olap hash table. #define THREAD_STACKSIZE (16 * 512 * 512) // The amount of stack space to allocate to each thread. #define VALID_FRAG 1 // Indicates fragment was valid in the fragment store //#define WINDOW_SCREEN_OLAP 10 // Amount by which k-mers can overlap a screen region and still // be added to the hash table. #define MAX_EXTRA_SUBCOUNT (AS_MAX_READLEN / G.Kmer_Len) #define HASH_FUNCTION(k) (((k) ^ ((k) >> HSF1) ^ ((k) >> HSF2)) & HASH_MASK) // Gives subscript in hash table for key k #define HASH_CHECK_FUNCTION(k) (((k) ^ ((k) >> SV1) ^ ((k) >> SV2)) & HASH_CHECK_MASK) // Gives bit position to see if key could be in bucket #define KEY_CHECK_FUNCTION(k) (((k) ^ ((k) >> SV1) ^ ((k) >> SV3)) & CHECK_MASK) // Gives bit pattern to see if key could match #define PROBE_FUNCTION(k) ((((k) ^ ((k) >> SV2) ^ ((k) >> SV3)) & PROBE_MASK) | 1) // Gives secondary hash function. Force to be odd so that will be relatively // prime wrt the hash table size, which is a power of 2. typedef enum Direction_Type { FORWARD, REVERSE } Direction_t; typedef struct String_Olap_Node { uint32 String_Num; // Of hash-table frag that have exact match with int32 Match_List; // Subscript of start of list of exact matches double diag_sum; // Sum of diagonals of all k-mer matches to this frag int32 diag_ct; // Count of all k-mer matches to this frag signed int Next : 29; // Next match if this is a collision unsigned Full : 1; unsigned consistent : 1; } String_Olap_t; typedef struct Olap_Info { int s_lo, s_hi; int t_lo, t_hi; double quality; int delta [AS_MAX_READLEN+1]; // needs only MAX_ERRORS int delta_ct; int s_left_boundary, s_right_boundary; int t_left_boundary, t_right_boundary; int min_diag, max_diag; } Olap_Info_t; // The following structure holds what used to be global information, but // is now encapsulated so that multiple copies can be made for multiple // parallel threads. typedef struct Work_Area { //int *Error_Bound; String_Olap_t * String_Olap_Space; int32 String_Olap_Size; int32 Next_Avail_String_Olap; Match_Node_t * Match_Node_Space; int32 Match_Node_Size; int32 Next_Avail_Match_Node; // Counts the number of overlaps for each fragment. Cuts off // overlaps above a limit. int32 A_Olaps_For_Frag; int32 B_Olaps_For_Frag; gkStore *gkpStore; int left_end_screened; int right_end_screened; int status; int thread_id; uint32 bgnID; // Range of reads we are processing uint32 endID; // was frag_segment_lo and frag_segment_hi (all lowercase) // Instead of outputting each overlap as we create it, we // buffer them and output blocks of overlaps. uint64 overlapsLen; uint64 overlapsMax; ovOverlap *overlaps; // Various stats that used to be global and updated whenever we // output an overlap or finished processing a set of hits. // Needed a mutex to update. uint64 Total_Overlaps; uint64 Contained_Overlap_Ct; uint64 Dovetail_Overlap_Ct; uint64 Kmer_Hits_Without_Olap_Ct; uint64 Kmer_Hits_With_Olap_Ct; uint64 Multi_Overlap_Ct; prefixEditDistance *editDist; char * q_diff; Olap_Info_t *distinct_olap; } Work_Area_t; typedef uint32 Check_Vector_t; // Bit vector to see if hash bucket could possibly contain a match typedef uint64 String_Ref_t; #define BIT_EMPT 62 #define BIT_LAST 63 extern uint32 STRING_NUM_BITS; extern uint32 OFFSET_BITS; extern uint64 TRUELY_ZERO; extern uint64 TRUELY_ONE; extern uint64 STRING_NUM_MASK; extern uint64 OFFSET_MASK; extern uint64 MAX_STRING_NUM; // // [ Last (1) ][ Empty (1) ][ Offset (11) ][ StringNum (19) ] // #define getStringRefStringNum(X) (((X) ) & STRING_NUM_MASK) #define getStringRefOffset(X) (((X) >> STRING_NUM_BITS) & OFFSET_MASK) #define getStringRefEmpty(X) (((X) >> BIT_EMPT ) & TRUELY_ONE) #define getStringRefLast(X) (((X) >> BIT_LAST ) & TRUELY_ONE) #define setStringRefStringNum(X, Y) ((X) = (((X) & ~(STRING_NUM_MASK )) | ((Y)))) #define setStringRefOffset(X, Y) ((X) = (((X) & ~(OFFSET_MASK << STRING_NUM_BITS)) | ((Y) << STRING_NUM_BITS))) #define setStringRefEmpty(X, Y) ((X) = (((X) & ~(TRUELY_ONE << BIT_EMPT )) | ((Y) << BIT_EMPT))) #define setStringRefLast(X, Y) ((X) = (((X) & ~(TRUELY_ONE << BIT_LAST )) | ((Y) << BIT_LAST))) typedef struct Hash_Bucket { String_Ref_t Entry [ENTRIES_PER_BUCKET]; unsigned char Check [ENTRIES_PER_BUCKET]; unsigned char Hits [ENTRIES_PER_BUCKET]; int16 Entry_Ct; } Hash_Bucket_t; typedef struct Hash_Frag_Info { uint32 length : 30; uint32 lfrag_end_screened : 1; uint32 rfrag_end_screened : 1; } Hash_Frag_Info_t; extern char *basesData; extern char *qualsData; extern String_Ref_t *nextRef; extern size_t Data_Len; extern int64 Bad_Short_Window_Ct; extern int64 Bad_Long_Window_Ct; extern size_t Extra_Data_Len; extern uint64 Max_Extra_Ref_Space; extern uint64 Extra_Ref_Ct; extern String_Ref_t * Extra_Ref_Space; extern uint64 Extra_String_Ct; extern uint64 Extra_String_Subcount; extern Check_Vector_t * Hash_Check_Array; extern uint64 Hash_String_Num_Offset; extern Hash_Bucket_t * Hash_Table; extern uint64 Kmer_Hits_With_Olap_Ct; extern uint64 Kmer_Hits_Without_Olap_Ct; extern uint64 Multi_Overlap_Ct; extern uint64 String_Ct; extern Hash_Frag_Info_t * String_Info; extern int64 * String_Start; extern uint32 String_Start_Size; extern size_t Used_Data_Len; extern int32 Bit_Equivalent [256]; extern int32 Char_Is_Bad [256]; extern uint64 Hash_Entries; extern uint64 Total_Overlaps; extern uint64 Contained_Overlap_Ct; extern uint64 Dovetail_Overlap_Ct; class oicParameters { public: oicParameters() {}; ~oicParameters() {}; void initialize(void) { maxErate = 0.06; Doing_Partial_Overlaps = false; bgnHashID = 1; endHashID = UINT32_MAX; minLibToHash = 0; maxLibToHash = UINT32_MAX; bgnRefID = 1; endRefID = UINT32_MAX; minLibToRef = 0; maxLibToRef = UINT32_MAX; Kmer_Len = 0; Kmer_Skip_File = NULL; Frag_Olap_Limit = UINT64_MAX; Unique_Olap_Per_Pair = true; Hash_Mask_Bits = 22; Max_Hash_Load = 0.6; Max_Hash_Strings = 10000; Max_Hash_Data_Len = 100000000; Outfile_Name = NULL; Outstat_Name = NULL; Num_PThreads = 1; Min_Olap_Len = 0; Use_Window_Filter = false; Use_Hopeless_Check = true; Frag_Store_Path = NULL; }; double maxErate; // If set true by the G option (G for Granger) // then allow overlaps that do not extend to the end // of either read. bool Doing_Partial_Overlaps; // -G uint32 bgnHashID; // -h uint32 endHashID; uint32 minLibToHash; // -H uint32 maxLibToHash; // The range of reads we need to process uint32 frag_segment_lo; uint32 frag_segment_hi; uint32 bgnRefID; // -r uint32 curRefID; // When processing, this is where we are at, bgn < cur <= end. uint32 endRefID; uint32 minLibToRef; // -R uint32 maxLibToRef; uint32 perThread; // When processing, how many to do per block uint64 Kmer_Len; // -k FILE *Kmer_Skip_File; // -k // Maximum number of overlaps for end of an old fragment against // a single hash table of frags, in each orientation uint64 Frag_Olap_Limit; // -l // If true will allow at most // one overlap output message per oriented fragment pair // Set true by -u command-line option; set false by -m bool Unique_Olap_Per_Pair; // -m and -u uint32 Hash_Mask_Bits; // --hashbits uint32 Max_Hash_Strings; // --hashstrings uint64 Max_Hash_Data_Len; // --hashdatalen double Max_Hash_Load; // --hashload // --maxreadlen sets OFFSET_BITS, STRING_NUM_BITS, STRING_NUM_MASK and MAX_STRING_NUM. char *Outfile_Name; // -o char *Outstat_Name; // -s uint32 Num_PThreads; // -t int32 Min_Olap_Len; // --minlength, former -v // Determines whether check for a window containing too many // errors is used to disqualify overlaps. bool Use_Window_Filter; // -w // Determines whether check for absence of kmer matches // at the end of a read is used to abort the overlap before // the extension from a single kmer match is attempted. bool Use_Hopeless_Check; // -z char *Frag_Store_Path; }; extern oicParameters G; extern uint64 HSF1; extern uint64 HSF2; extern uint64 SV1; extern uint64 SV2; extern uint64 SV3; extern ovFile *Out_BOF; extern pthread_mutex_t Frag_Store_Mutex; extern pthread_mutex_t Write_Proto_Mutex; void Output_Overlap(uint32 S_ID, int S_Len, Direction_t S_Dir, uint32 T_ID, int T_Len, Olap_Info_t * olap, Work_Area_t *WA); void Output_Partial_Overlap(uint32 s_id, uint32 t_id, Direction_t dir, const Olap_Info_t * p, int s_len, int t_len, Work_Area_t *WA); int Process_String_Olaps (char * S, int Len, char * S_quality, uint32 ID, Direction_t Dir, Work_Area_t * WA); void Find_Overlaps (char Frag [], int Frag_Len, char quality [], uint32 Frag_Num, Direction_t Dir, Work_Area_t * WA); void * Process_Overlaps (void *); int Build_Hash_Index(gkStore *store, uint32 bgnID, uint32 endID); #endif // OVERLAPINCORE_H canu-1.3/src/overlapInCore/overlapInCore.mk000066400000000000000000000013251272335422300207470ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := overlapInCore SOURCES := overlapInCore.C \ overlapInCore-Build_Hash_Index.C \ overlapInCore-Find_Overlaps.C \ overlapInCore-Output.C \ overlapInCore-Process_Overlaps.C \ overlapInCore-Process_String_Overlaps.C SRC_INCDIRS := .. ../AS_UTL ../stores liboverlap TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/overlapInCore/overlapInCorePartition.C000066400000000000000000000333071272335422300224210ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVL/overlap_partition.C * * Modifications by: * * Brian P. Walenz from 2011-JUN-12 to 2013-AUG-01 * are Copyright 2011-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2012-JUL-29 to 2013-DEC-02 * are Copyright 2012-2013 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz from 2014-NOV-21 to 2015-AUG-25 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "AS_UTL_decodeRange.H" // Reads gkpStore, outputs three files: // ovlbat - batch names // ovljob - job names // ovlopt - overlapper options uint32 batchMax = 1000; void outputJob(FILE *BAT, FILE *JOB, FILE *OPT, uint32 hashBeg, uint32 hashEnd, uint32 refBeg, uint32 refEnd, uint32 maxNumReads, uint32 maxLength, uint32 &batchSize, uint32 &batchName, uint32 &jobName) { fprintf(BAT, "%03"F_U32P"\n", batchName); fprintf(JOB, "%06"F_U32P"\n", jobName); if (maxNumReads == 0) { fprintf(OPT, "-h "F_U32"-"F_U32" -r "F_U32"-"F_U32"\n", hashBeg, hashEnd, refBeg, refEnd); fprintf(stderr, "HASH %10d-%10d REFR %10d-%10d JOB %d\n", hashBeg, hashEnd, refBeg, refEnd, jobName); } else { fprintf(OPT, "-h "F_U32"-"F_U32" -r "F_U32"-"F_U32" --hashstrings "F_U32" --hashdatalen "F_U32"\n", hashBeg, hashEnd, refBeg, refEnd, maxNumReads, maxLength); fprintf(stderr, "HASH %10d-%10d REFR %10d-%10d STRINGS %10d BASES %10d JOB %d\n", hashBeg, hashEnd, refBeg, refEnd, maxNumReads, maxLength, jobName); } refBeg = refEnd + 1; batchSize++; if (batchSize >= batchMax) { batchSize = 0; batchName++; } jobName++; } uint32 * loadReadLengths(gkStore *gkp, set &libToHash, uint32 &hashMin, uint32 &hashMax, set &libToRef, uint32 &refMin, uint32 &refMax) { uint32 numReads = gkp->gkStore_getNumReads(); uint32 numLibs = gkp->gkStore_getNumLibraries(); uint32 *readLen = new uint32 [numReads + 1]; bool testHash = false; bool testRef = false; if (libToHash.size() > 0) { testHash = true; hashMin = UINT32_MAX; hashMax = 0; } if (libToRef.size() > 0) { testRef = true; refMin = UINT32_MAX; refMax = 0; } bool *doHash = new bool [numLibs + 1]; bool *doRef = new bool [numLibs + 1]; for (uint32 i=0; i<=numLibs; i++) { doHash[i] = (libToHash.count(i) == 0) ? false : true; doRef[i] = (libToRef.count(i) == 0) ? false : true; } fprintf(stderr, "Loading lengths of "F_U32" fragments ("F_SIZE_T"mb)\n", numReads, (numReads * sizeof(uint32)) >> 20); memset(readLen, 0, sizeof(uint32) * (numReads + 1)); for (uint32 ii=1; ii<=numReads; ii++) { gkRead *read = gkp->gkStore_getRead(ii); if (read->gkRead_readID() != ii) fprintf(stderr, "ERROR: readID=%u != ii=%u\n", read->gkRead_readID(), ii); assert(read->gkRead_readID() == ii); readLen[ii] = read->gkRead_sequenceLength(); if ((testHash == true) && (doHash[read->gkRead_libraryID()] == true)) { if (ii < hashMin) hashMin = ii; if (hashMax < ii) hashMax = ii; } if ((testRef == true) && (doRef[read->gkRead_libraryID()] == true)) { if (ii < refMin) refMin = ii; if (refMax < ii) refMax = ii; } if ((ii % 1048576) == 0) fprintf(stderr, "Loading lengths at "F_U32" out of "F_U32". H: "F_U32","F_U32" R: "F_U32","F_U32"\n", ii, numReads, hashMin, hashMax, refMin, refMax); } return(readLen); } void partitionFrags(gkStore *gkp, FILE *BAT, FILE *JOB, FILE *OPT, uint32 minOverlapLength, uint64 ovlHashBlockSize, uint64 ovlRefBlockLength, uint64 ovlRefBlockSize, set &libToHash, set &libToRef) { uint32 hashMin = 1; uint32 hashBeg = 1; uint32 hashEnd = 0; uint32 hashMax = UINT32_MAX; uint32 refMin = 1; uint32 refBeg = 1; uint32 refEnd = 0; uint32 refMax = UINT32_MAX; uint32 batchSize = 0; uint32 batchName = 1; uint32 jobName = 1; uint32 numReads = gkp->gkStore_getNumReads(); uint32 *readLen = NULL; if ((ovlRefBlockLength > 0) || (libToHash.size() > 0) || (libToRef.size() > 0)) readLen = loadReadLengths(gkp, libToHash, hashMin, hashMax, libToRef, refMin, refMax); if (hashMax > numReads) hashMax = numReads; if (refMax > numReads) refMax = numReads; fprintf(stderr, "Partitioning for hash: "F_U32"-"F_U32" ref: "F_U32","F_U32"\n", hashMin, hashMax, refMin, refMax); hashBeg = hashMin; while (hashBeg < hashMax) { hashEnd = hashBeg + ovlHashBlockSize - 1; if (hashEnd > hashMax) hashEnd = hashMax; refBeg = refMin; refEnd = 0; while ((refBeg < refMax) && ((refBeg < hashEnd) || (libToHash.size() != 0 && libToHash == libToRef))) { uint64 refLen = 0; if (ovlRefBlockLength > 0) { do { refEnd++; if (readLen[refEnd] < minOverlapLength) continue; refLen += readLen[refEnd]; } while ((refLen < ovlRefBlockLength) && (refEnd < refMax)); } else { refEnd = refBeg + ovlRefBlockSize - 1; } if (refEnd > refMax) refEnd = refMax; if ((refEnd > hashEnd) && (libToHash.size() == 0 || libToHash != libToRef)) refEnd = hashEnd; outputJob(BAT, JOB, OPT, hashBeg, hashEnd, refBeg, refEnd, 0, 0, batchSize, batchName, jobName); refBeg = refEnd + 1; } hashBeg = hashEnd + 1; } } void partitionLength(gkStore *gkp, FILE *BAT, FILE *JOB, FILE *OPT, uint32 minOverlapLength, uint64 ovlHashBlockLength, uint64 ovlRefBlockLength, uint64 ovlRefBlockSize, set &libToHash, set &libToRef) { uint32 hashMin = 1; uint32 hashBeg = 1; uint32 hashEnd = 0; uint32 hashMax = UINT32_MAX; uint32 refMin = 1; uint32 refBeg = 1; uint32 refEnd = 0; uint32 refMax = UINT32_MAX; uint32 batchSize = 0; uint32 batchName = 1; uint32 jobName = 1; uint32 numReads = gkp->gkStore_getNumReads(); uint32 *readLen = loadReadLengths(gkp, libToHash, hashMin, hashMax, libToRef, refMin, refMax); if (hashMax > numReads) hashMax = numReads; if (refMax > numReads) refMax = numReads; fprintf(stderr, "Partitioning for hash: "F_U32"-"F_U32" ref: "F_U32","F_U32"\n", hashMin, hashMax, refMin, refMax); hashBeg = hashMin; hashEnd = hashMin - 1; while (hashBeg < hashMax) { uint64 hashLen = 0; assert(hashEnd == hashBeg - 1); // Non deleted reads contribute one byte per untrimmed base, and every fragment contributes one // more byte for the terminating zero. In canu, there are no deleted reads. do { hashEnd++; if (readLen[hashEnd] < minOverlapLength) continue; hashLen += readLen[hashEnd] + 1; } while ((hashLen < ovlHashBlockLength) && (hashEnd < hashMax)); assert(hashEnd <= hashMax); refBeg = refMin; refEnd = 0; while ((refBeg < refMax) && ((refBeg < hashEnd) || (libToHash.size() != 0 && libToHash == libToRef))) { uint64 refLen = 0; if (ovlRefBlockLength > 0) { do { refEnd++; if (readLen[refEnd] < minOverlapLength) continue; refLen += readLen[refEnd]; } while ((refLen < ovlRefBlockLength) && (refEnd < refMax)); } else { refEnd = refBeg + ovlRefBlockSize - 1; } if (refEnd > refMax) refEnd = refMax; if ((refEnd > hashEnd) && (libToHash.size() == 0 || libToHash != libToRef)) refEnd = hashEnd; outputJob(BAT, JOB, OPT, hashBeg, hashEnd, refBeg, refEnd, hashEnd - hashBeg + 1, hashLen, batchSize, batchName, jobName); refBeg = refEnd + 1; } hashBeg = hashEnd + 1; } } int main(int argc, char **argv) { char *gkpStoreName = NULL; gkStore *gkpStore = NULL; char *outputPrefix = NULL; char outputName[FILENAME_MAX]; uint64 ovlHashBlockLength = 0; uint64 ovlHashBlockSize = 0; uint64 ovlRefBlockLength = 0; uint64 ovlRefBlockSize = 0; uint32 minOverlapLength = 0; bool checkAllLibUsed = true; set libToHash; set libToRef; AS_configure(argc, argv); int arg = 1; int err = 0; while (arg < argc) { if (strcmp(argv[arg], "-g") == 0) { gkpStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-bl") == 0) { ovlHashBlockLength = strtoull(argv[++arg], NULL, 10); } else if (strcmp(argv[arg], "-bs") == 0) { ovlHashBlockSize = strtoull(argv[++arg], NULL, 10); } else if (strcmp(argv[arg], "-rl") == 0) { ovlRefBlockLength = strtoull(argv[++arg], NULL, 10); } else if (strcmp(argv[arg], "-rs") == 0) { ovlRefBlockSize = strtoull(argv[++arg], NULL, 10); } else if (strcmp(argv[arg], "-ol") == 0) { minOverlapLength = strtoull(argv[++arg], NULL, 10); } else if (strcmp(argv[arg], "-H") == 0) { AS_UTL_decodeRange(argv[++arg], libToHash); } else if (strcmp(argv[arg], "-R") == 0) { AS_UTL_decodeRange(argv[++arg], libToRef); } else if (strcmp(argv[arg], "-C") == 0) { checkAllLibUsed = false; } else if (strcmp(argv[arg], "-o") == 0) { outputPrefix = argv[++arg]; } else { fprintf(stderr, "ERROR: Unknown option '%s'\n", arg[argv]); err++; } arg++; } if (err) { fprintf(stderr, "usage: %s [opts]\n", argv[0]); exit(1); } if ((ovlHashBlockLength > 0) && (ovlHashBlockSize > 0)) fprintf(stderr, "ERROR: At most one of -bl and -bs can be non-zero.\n"), exit(1); if ((ovlRefBlockLength > 0) && (ovlRefBlockSize > 0)) fprintf(stderr, "ERROR: At most one of -rl and -rs can be non-zero.\n"), exit(1); fprintf(stderr, "HASH: "F_U64" reads or "F_U64" length.\n", ovlHashBlockSize, ovlHashBlockLength); fprintf(stderr, "REF: "F_U64" reads or "F_U64" length.\n", ovlRefBlockSize, ovlRefBlockLength); gkStore *gkp = gkStore::gkStore_open(gkpStoreName); uint32 numLibs = gkp->gkStore_getNumLibraries(); uint32 invalidLibs = 0; for (set::iterator it=libToHash.begin(); it != libToHash.end(); it++) if (numLibs < *it) fprintf(stderr, "ERROR: -H "F_U32" is invalid; only "F_U32" libraries in '%s'\n", *it, numLibs, gkpStoreName), invalidLibs++; for (set::iterator it=libToRef.begin(); it != libToRef.end(); it++) if (numLibs < *it) fprintf(stderr, "ERROR: -R "F_U32" is invalid; only "F_U32" libraries in '%s'\n", *it, numLibs, gkpStoreName), invalidLibs++; if ((libToHash.size() > 0) && (libToRef.size() > 0)) { for (uint32 lib=1; lib<=numLibs; lib++) { if ((libToHash.find(lib) == libToHash.end()) && (libToRef.find(lib) == libToRef.end())) { if (checkAllLibUsed == true) fprintf(stderr, "ERROR: library "F_U32" is not mentioned in either -H or -R.\n", lib), invalidLibs++; else fprintf(stderr, "Warning: library "F_U32" is not mentioned in either -H or -R.\n", lib); } } } if (invalidLibs > 0) fprintf(stderr, "ERROR: one of -H and/or -R are invalid.\n"), exit(1); errno = 0; sprintf(outputName, "%s.ovlbat", outputPrefix); FILE *BAT = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", outputName, strerror(errno)), exit(1); sprintf(outputName, "%s.ovljob", outputPrefix); FILE *JOB = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", outputName, strerror(errno)), exit(1); sprintf(outputName, "%s.ovlopt", outputPrefix); FILE *OPT = fopen(outputName, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", outputName, strerror(errno)), exit(1); if (ovlHashBlockLength == 0) partitionFrags(gkp, BAT, JOB, OPT, minOverlapLength, ovlHashBlockSize, ovlRefBlockLength, ovlRefBlockSize, libToHash, libToRef); else partitionLength(gkp, BAT, JOB, OPT, minOverlapLength, ovlHashBlockLength, ovlRefBlockLength, ovlRefBlockSize, libToHash, libToRef); fclose(BAT); fclose(JOB); fclose(OPT); gkp->gkStore_close(); exit(0); } canu-1.3/src/overlapInCore/overlapInCorePartition.mk000066400000000000000000000010021272335422300226310ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := overlapInCorePartition SOURCES := overlapInCorePartition.C SRC_INCDIRS := .. ../AS_UTL ../stores liboverlap TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/overlapInCore/overlapPair.C000066400000000000000000000523051272335422300202420ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAR-27 to 2015-JUL-20 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-FEB-12 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include #include "gkStore.H" #include "ovStore.H" //#define BUSTED #define FALCON #ifdef FALCON #include "dw.H" #else #include "ssw_cpp.H" #endif #ifdef BUSTED #include "NDalign.H" #endif #include "overlapReadCache.H" #include "AS_UTL_reverseComplement.H" #include "timeAndSize.H" // getTime(); // The process will load BATCH_SIZE overlaps into memory, then load all the reads referenced by // those overlaps. Once all data is loaded, compute threads are spawned. Each thread will reserve // THREAD_SIZE overlaps to compute. A small THREAD_SIZE relative to BATCH_SIZE will result in // better load balancing, but too small and the overhead of reserving overlaps will dominate (too // small is on the order of 1). While threads are computing, the next batch of overlaps and reads // is loaded. // // A large BATCH_SIZE will make startup cost large - no computes are started until the initial load // is finished. To alleivate this (a little bit), the initial load is only 1/8 of the full // BATCH_SIZE. #define BATCH_SIZE 1024 * 1024 #define THREAD_SIZE 128 #ifdef FALCON // we don't alow slop because the ND aligner in falcon has no concept of non-global aligns. It always slams reads in so we find the conservative overlap and then we extend if it's close to a dovetail manually #define MHAP_SLOP 0 #else #define MHAP_SLOP 500 #endif overlapReadCache *rcache = NULL; // Used to be just 'cache', but that conflicted with -pg: /usr/lib/libc_p.a(msgcat.po):(.bss+0x0): multiple definition of `cache' uint32 batchPrtID = 0; // When to report progress uint32 batchPosID = 0; // The current position of the batch uint32 batchEndID = 0; // The end of the batch pthread_mutex_t balanceMutex; class workSpace { public: workSpace() { threadID = 0; maxErate = 0; partialOverlaps = false; invertOverlaps = false; gkpStore = NULL; align = NULL; //analyze = NULL; overlapsLen = 0; overlaps = NULL; readSeq = NULL; }; ~workSpace() { #ifdef BUSTED delete NDaln; NDaln = NULL; #endif #ifndef FALCON delete align; delete filter; align = filter = NULL; #else delete align; align=NULL; #endif delete[] readSeq; }; public: uint32 threadID; double maxErate; bool partialOverlaps; bool invertOverlaps; char* readSeq; gkStore *gkpStore; #ifdef BUSTED NDalign *NDaln; #endif #ifndef FALCON StripedSmithWaterman::Aligner *align; StripedSmithWaterman::Filter *filter; #else NDalignment::NDalignResult *align; #endif //analyzeAlignment *analyze; uint32 overlapsLen; // Not used. ovOverlap *overlaps; }; bool getRange(uint32 &bgnID, uint32 &endID) { pthread_mutex_lock(&balanceMutex); bgnID = batchPosID; batchPosID += THREAD_SIZE; // Supposed to overflow. endID = batchPosID; if (endID > batchEndID) endID = batchEndID; #if 0 if (batchPosID >= batchPrtID) { fprintf(stderr, "getRange()-- at %u out of %u -- %6.2f%%\n", batchPosID, batchEndID, 100.0 * batchPosID / batchEndID); batchPrtID += batchEndID / 32; } #endif pthread_mutex_unlock(&balanceMutex); // If we're out of overlaps, batchPosID is more than batchEndID (from the last call to this // function), which makes bgnID > endID (in this call). return(bgnID < endID); } void * recomputeOverlaps(void *ptr) { workSpace *WA = (workSpace *)ptr; uint32 bgnID = 0; uint32 endID = 0; uint32 nPassed = 0; uint32 nFailed = 0; uint32 nTested = 0; // Lazy allocation of the prefixEditDistance structure; it's slow. #ifdef BUSTED if (WA->NDaln == NULL) WA->NDaln = new NDalign(WA->partialOverlaps ? pedLocal : pedOverlap, WA->maxErate, 15); #endif //if (WA->analyze == NULL) // WA->analyze = new analyzeAlignment(); while (getRange(bgnID, endID)) { //fprintf(stderr, "Thread %2u computes overlaps %7u - %7u\n", WA->threadID, bgnID, endID); double startTime = getTime(); for (uint32 oo=bgnID; oooverlaps + oo; if (WA->invertOverlaps) { ovOverlap swapped = WA->overlaps[oo]; WA->overlaps[oo].swapIDs(swapped); // Needs to be from a temporary! } #if 0 fprintf(stderr, "BEGIN overlap A %5u %5u-%5u B %5u %5u-%5u\n", ovl->a_iid, ovl->a_bgn(), ovl->b_end(), ovl->a_iid, ovl->b_bgn(), ovl->b_end()); #endif // This closely follows readConsensus #if 0 if (ovl->b_iid != 64) continue; #endif // Invalidate the overlap. ovl->evalue(AS_MAX_EVALUE); uint32 aID = ovl->a_iid; uint32 bID = ovl->b_iid; // Compute the overlap #if 0 nTested++; if (nTested % 1000 == 0) { double deltaTime = getTime() - startTime; fprintf(stderr, "*******Thread %2u computed overlaps %7u - %7u in %7.3f seconds - %6.2f olaps per second (%8u fail %8u pass)\n", WA->threadID, bgnID, endID, deltaTime, (endID - bgnID) / deltaTime, nFailed, nPassed); } #endif #ifdef BUSTED WA->NDaln->initialize(aID, rcache->getRead(aID), rcache->getLength(aID), ovl->a_bgn(), ovl->a_end(), bID, rcache->getRead(bID), rcache->getLength(bID), ovl->b_bgn(), ovl->b_end(), ovl->flipped()); if (WA->NDaln->findMinMaxDiagonal(40) == false) { fprintf(stderr, "A %6u %5d-%5d -> B %6u %5d-%5d %s ALIGN LENGTH TOO SHORT.\n", aID, ovl->a_bgn(), ovl->a_end(), bID, ovl->b_bgn(), ovl->b_end(), ovl->flipped() ? "<-" : "->"); continue; } if (WA->NDaln->findSeeds(true) == false) { fprintf(stderr, "A %6u %5d-%5d -> B %6u %5d-%5d %s NO SEEDS.\n", aID, ovl->a_bgn(), ovl->a_end(), bID, ovl->b_bgn(), ovl->b_end(), ovl->flipped() ? "<-" : "->"); continue; } if ((WA->NDaln->findHits() == true) && (WA->NDaln->chainHits() == true) && (WA->NDaln->processHits() == true)) { WA->align->display("MHAP align():", true); // fprintf(stderr, "Reads %d to %d, expected overlap %d - %d to %d - %d and found error rate %f from %d - %d and %d - %d\n", aID, bID, ovl->a_bgn(), ovl->a_end(), ovl->b_bgn(), ovl->b_end(), WA->NDaln->erate(), WA->NDaln->abgn(), WA->NDaln->aend(), WA->NDaln->bbgn(), WA->NDaln->bend()); #else char *bRead = WA->readSeq; int32 astart = std::max((int32)0, (int32)ovl->a_bgn() - MHAP_SLOP); int32 aend = std::min((int32)rcache->getLength(aID), (int32)ovl->a_end() + MHAP_SLOP); int32 bstart = std::max((int32)0, (int32)ovl->b_bgn() - MHAP_SLOP); int32 bend = std::min((int32)rcache->getLength(bID), (int32)ovl->b_end() + MHAP_SLOP); strcpy(bRead, rcache->getRead(bID)); if (ovl->flipped()) { reverseComplementSequence(bRead, rcache->getLength(bID)); bstart = std::max((int32)0, (int32)rcache->getLength(bID) - (int32)ovl->b_bgn() - MHAP_SLOP); bend = std::min((int32)rcache->getLength(bID), (int32)rcache->getLength(bID) - (int32)ovl->b_end() + MHAP_SLOP); } #ifdef FALCON int tolerance = std::min(150, (int)round(0.5 * WA->maxErate * (rcache->getLength(aID) + rcache->getLength(bID)))); WA->align->clear(); bool aligned = NDalignment::align(bRead+bstart, bend-bstart+1, rcache->getRead(aID)+astart, aend-astart+1, tolerance, false, *(WA->align)); NDalignment::NDalignResult& alignResult = *WA->align; uint32 alignmentLength = WA->align->_tgt_end - WA->align->_tgt_bgn + 1; #else StripedSmithWaterman::Alignment alignment; WA->align->Align(bRead+bstart, rcache->getRead(aID)+astart, aend-astart+1, *WA->filter, &alignment); NDalignment::NDalignResult alignResult; alignResult._dist = alignment.mismatches; alignResult._tgt_bgn = alignment.ref_begin; alignResult._tgt_end = alignment.ref_end; alignResult._qry_bgn = alignment.query_begin; alignResult._qry_end = alignment.query_end; uint32 alignmentLength = alignment.ref_end-alignment.ref_begin+1; #endif //fprintf(stderr, "Reads %d (%d) to %d (%d), expected overlap %d - %d to %d - %d and found error rate %f from %d - %d and %d - %d\n", aID, rcache->getLength(aID), bID, rcache->getLength(bID), ovl->a_bgn(), ovl->a_end(), ovl->b_bgn(), ovl->b_end(), (double)alignResult._dist/(alignmentLength),alignResult._tgt_bgn+astart, alignResult._tgt_end+astart-1, alignResult._qry_bgn+bstart, alignResult._qry_end+bstart-1); if (alignmentLength > 40 && ((double)alignResult._dist / (double) (alignmentLength)) < WA->maxErate) { #endif nPassed++; //WA->align->display(); #ifdef BUSTED ovl->dat.ovl.bhg5 = WA->NDaln->bhg5(); ovl->dat.ovl.bhg3 = WA->NDaln->bhg3(); ovl->dat.ovl.ahg5 = WA->NDaln->ahg5(); ovl->dat.ovl.ahg3 = WA->NDaln->ahg3(); ovl->erate(WA->NDaln->erate()); #else ovl->dat.ovl.ahg5 = alignResult._tgt_bgn+astart; ovl->dat.ovl.ahg3 = rcache->getLength(aID) - (alignResult._tgt_end+astart - 1); ovl->dat.ovl.bhg5 = alignResult._qry_bgn + bstart; ovl->dat.ovl.bhg3 = rcache->getLength(bID) - (alignResult._qry_end + bstart - 1); // check for almost dovetail if we're not looking for partial and extend if (WA->partialOverlaps == false) { if ((double)(alignResult._dist + ovl->dat.ovl.ahg5) / (alignmentLength+ovl->dat.ovl.ahg5) < WA->maxErate) { alignResult._dist += ovl->dat.ovl.ahg5; alignmentLength += ovl->dat.ovl.ahg5; ovl->dat.ovl.ahg5 = 0; } if ((double)(alignResult._dist + ovl->dat.ovl.ahg3) / (alignmentLength+ovl->dat.ovl.ahg3) < WA->maxErate) { alignResult._dist += ovl->dat.ovl.ahg3; alignmentLength += ovl->dat.ovl.ahg3; ovl->dat.ovl.ahg3 = 0; } if ((double)(alignResult._dist + ovl->dat.ovl.bhg5) / (alignmentLength+ovl->dat.ovl.bhg5) < WA->maxErate) { alignResult._dist += ovl->dat.ovl.bhg5; alignmentLength += ovl->dat.ovl.bhg5; ovl->dat.ovl.bhg5 = 0; } if ((double)(alignResult._dist + ovl->dat.ovl.bhg3) / (alignmentLength+ovl->dat.ovl.bhg3) < WA->maxErate) { alignResult._dist += ovl->dat.ovl.bhg3; alignmentLength += ovl->dat.ovl.bhg3; ovl->dat.ovl.bhg3 = 0; } } ovl->erate((double)alignResult._dist/(alignmentLength)); // fprintf(stderr, "Reads %d (%d) to %d (%d), updated overlap to be %d - %d to %d - %d at error rate %f\n", aID, rcache->getLength(aID), bID, rcache->getLength(bID), ovl->a_bgn(), ovl->a_end(), ovl->b_bgn(), ovl->b_end(), ovl->erate()); // #endif ovl->dat.ovl.forOBT = (WA->partialOverlaps == true); ovl->dat.ovl.forDUP = (WA->partialOverlaps == true); ovl->dat.ovl.forUTG = (WA->partialOverlaps == false) && (ovl->overlapIsDovetail() == true); } else { nFailed++; ovl->evalue(AS_MAX_EVALUE); ovl->dat.ovl.forOBT = false; ovl->dat.ovl.forDUP = false; ovl->dat.ovl.forUTG = false; } } #if 0 double deltaTime = getTime() - startTime; fprintf(stderr, "Thread %2u computed overlaps %7u - %7u in %7.3f seconds - %6.2f olaps per second (%8u fail %8u pass)\n", WA->threadID, bgnID, endID, deltaTime, (endID - bgnID) / deltaTime, nFailed, nPassed); #endif } // Report. The last batch has no work to do. if (nFailed + nPassed > 0) fprintf(stderr, "Thread %u finished -- %u failed %u passed.\n", WA->threadID, nFailed, nPassed); return(NULL); } int main(int argc, char **argv) { char *gkpName = NULL; char *ovlName = NULL; char *outName = NULL; uint32 bgnID = 0; uint32 endID = UINT32_MAX; uint32 numThreads = 1; double maxErate = 0.12; bool partialOverlaps = false; bool invertOverlaps = false; uint64 memLimit = 4; argc = AS_configure(argc, argv); int err=0; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { ovlName = argv[++arg]; } else if (strcmp(argv[arg], "-o") == 0) { outName = argv[++arg]; } else if (strcmp(argv[arg], "-b") == 0) { bgnID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-e") == 0) { endID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-t") == 0) { numThreads = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-erate") == 0) { maxErate = atof(argv[++arg]); } else if (strcmp(argv[arg], "-partial") == 0) { partialOverlaps = true; } else if (strcmp(argv[arg], "-invert") == 0) { invertOverlaps = true; } else if (strcmp(argv[arg], "-memory") == 0) { memLimit = atoi(argv[++arg]); } else { err++; } arg++; } if (gkpName == NULL) err++; if (ovlName == NULL) err++; if (outName == NULL) err++; if (err) { fprintf(stderr, "usage: %s ...\n", argv[0]); fprintf(stderr, " -G gkpStore Mandatory, path to gkpStore\n"); fprintf(stderr, "\n"); fprintf(stderr, "Inputs can come from either a store or a file.\n"); fprintf(stderr, " -O ovlStore \n"); fprintf(stderr, " -O ovlFile \n"); fprintf(stderr, "\n"); fprintf(stderr, "If from an ovlStore, the range of reads processed can be restricted.\n"); fprintf(stderr, " -b bgnID \n"); fprintf(stderr, " -e endID \n"); fprintf(stderr, "\n"); fprintf(stderr, "Outputs will be written to a store or file, depending on the input type\n"); fprintf(stderr, " -o ovlStore \n"); fprintf(stderr, " -o ovlFile \n"); fprintf(stderr, "\n"); fprintf(stderr, " -erate e Overlaps are computed at 'e' fraction error; must be larger than the original erate\n"); fprintf(stderr, " -partial Overlaps are 'overlapInCore -G' partial overlaps\n"); fprintf(stderr, " -memory m Use up to 'm' GB of memory\n"); fprintf(stderr, "\n"); fprintf(stderr, " -t n Use up to 'n' cores\n"); fprintf(stderr, "\n"); fprintf(stderr, "Advanced options:\n"); fprintf(stderr, "\n"); fprintf(stderr, " -invert Invert the overlap A <-> B before aligning (they are not re-inverted before output)\n"); fprintf(stderr, "\n"); exit(1); } gkStore *gkpStore = gkStore::gkStore_open(gkpName); ovStore *ovlStore = NULL, *ovlStoreOut = NULL; ovFile *ovlFile = NULL, *ovlFileOut = NULL; if (AS_UTL_fileExists(ovlName, true)) { fprintf(stderr, "Reading overlaps from store '%s' and writing to '%s'\n", ovlName, outName); ovlStore = new ovStore(ovlName, gkpStore); ovlStoreOut = new ovStore(outName, gkpStore, ovStoreWrite); if (bgnID < 1) bgnID = 1; if (endID > gkpStore->gkStore_getNumReads()) endID = gkpStore->gkStore_getNumReads(); ovlStore->setRange(bgnID, endID); } else { fprintf(stderr, "Reading overlaps from file '%s' and writing to '%s'\n", ovlName, outName); ovlFile = new ovFile(ovlName, ovFileFull); ovlFileOut = new ovFile(outName, ovFileFullWrite); } workSpace *WA = new workSpace [numThreads]; pthread_t *tID = new pthread_t [numThreads]; pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setstacksize(&attr, 12 * 131072); pthread_mutex_init(&balanceMutex, NULL); // Initialize thread work areas. Mirrored from overlapInCore.C for (uint32 tt=0; ttreadOverlaps(overlaps, overlapsMax, false); if (ovlFile) *overlapsLen = ovlFile->readOverlaps(overlaps, overlapsMax); overlapsMax *= 8; // Back to the normal batch size. fprintf(stderr, "Loaded %u overlaps.\n", *overlapsLen); rcache->loadReads(overlaps, *overlapsLen); // Loop over all the overlaps. while (overlapsALen + overlapsBLen > 0) { // Launch next batch of threads //fprintf(stderr, "LAUNCH THREADS\n"); // Globals, ugh. These limit the threads to the range of overlaps we have loaded. Each thread // will pull out THREAD_SIZE overlaps at a time to compute, updating batchPosID as it does so. // Each thread will stop when batchPosID > batchEndID. batchPrtID = 0; batchPosID = 0; batchEndID = *overlapsLen; for (uint32 tt=0; ttwriteOverlap(overlaps + oo); if (ovlFile) ovlFileOut->writeOverlaps(overlaps, *overlapsLen); // Load more overlaps if (ovlStore) *overlapsLen = ovlStore->readOverlaps(overlaps, overlapsMax, false); if (ovlFile) *overlapsLen = ovlFile->readOverlaps(overlaps, overlapsMax); fprintf(stderr, "Loaded %u overlaps.\n", *overlapsLen); rcache->loadReads(overlaps, *overlapsLen); // Wait for threads to finish for (uint32 tt=0; ttpurgeReads(); } // Goodbye. delete rcache; gkpStore->gkStore_close(); delete ovlStore; delete ovlStoreOut; delete ovlFile; delete ovlFileOut; delete [] overlapsA; delete [] overlapsB; delete [] WA; delete [] tID; return(0); } canu-1.3/src/overlapInCore/overlapPair.mk000066400000000000000000000010471272335422300204640ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := overlapPair SOURCES := overlapPair.C SRC_INCDIRS := .. ../AS_UTL ../stores ../meryl/libleaff liboverlap ../utgcns/libNDFalcon TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lleaff -lcanu TGT_PREREQS := libleaff.a libcanu.a SUBMAKEFILES := canu-1.3/src/overlapInCore/overlapReadCache.C000066400000000000000000000115271272335422300211470ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-JUN-25 to 2015-JUL-01 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "overlapReadCache.H" #include #include #include using namespace std; overlapReadCache::overlapReadCache(gkStore *gkpStore_, uint64 memLimit) { gkpStore = gkpStore_; nReads = gkpStore->gkStore_getNumReads(); readAge = new uint32 [nReads + 1]; readLen = new uint32 [nReads + 1]; memset(readAge, 0, sizeof(uint32) * (nReads + 1)); memset(readLen, 0, sizeof(uint32) * (nReads + 1)); readSeqFwd = new char * [nReads + 1]; //readSeqRev = new char * [nReads + 1]; memset(readSeqFwd, 0, sizeof(char *) * (nReads + 1)); //memset(readSeqRev, 0, sizeof(char *) * (nReads + 1)); memoryLimit = memLimit * 1024 * 1024 * 1024; } overlapReadCache::~overlapReadCache() { delete [] readAge; delete [] readLen; for (uint32 rr=0; rr<=nReads; rr++) { delete [] readSeqFwd[rr]; //delete [] readSeqRev[rr]; } delete [] readSeqFwd; //delete [] readSeqRev; } void overlapReadCache::loadRead(uint32 id) { gkRead *read = gkpStore->gkStore_getRead(id); gkpStore->gkStore_loadReadData(read, &readdata); readLen[id] = read->gkRead_sequenceLength(); readSeqFwd[id] = new char [readLen[id] + 1]; //readSeqRev[id] = new char [readLen[id] + 1]; memcpy(readSeqFwd[id], readdata.gkReadData_getSequence(), sizeof(char) * readLen[id]); readSeqFwd[id][readLen[id]] = 0; } // Make sure that the reads in 'reads' are in the cache. // Ideally, these are just the reads we need to load. void overlapReadCache::loadReads(set reads) { uint32 nn = 0; uint32 nc = reads.size() / 25; // For each read in the input set, load it. //if (reads.size() > 0) // fprintf(stderr, "loadReads()-- Need to load %u reads.\n", reads.size()); for (set::iterator it=reads.begin(); it != reads.end(); ++it) { //if ((++nn % nc) == 0) // fprintf(stderr, "loadReads()-- %6.2f%% finished.\n", 100.0 * nn / reads.size()); if (readLen[*it] != 0) continue; loadRead(*it); } //fprintf(stderr, "loadReads()-- %6.2f%% finished.\n", 100.0); // Age all the reads in the cache. uint32 nLoaded = 0; for (uint32 id=0; id 0) nLoaded++; readAge[id]++; } //fprintf(stderr, "loadReads()-- loaded %u -- %u in cache\n", reads.size(), nLoaded); } void overlapReadCache::markForLoading(set &reads, uint32 id) { // Note that it was just used. readAge[id] = 0; // Already loaded? Done! if (readLen[id] != 0) return; // Already pending? Done! if (reads.count(id) != 0) return; // Mark it for loading. reads.insert(id); } void overlapReadCache::loadReads(ovOverlap *ovl, uint32 nOvl) { set reads; for (uint32 oo=0; oo reads; markForLoading(reads, tig->tigID()); for (uint32 oo=0; oonumberOfChildren(); oo++) if (tig->getChild(oo)->isRead() == true) markForLoading(reads, tig->getChild(oo)->ident()); loadReads(reads); } void overlapReadCache::purgeReads(void) { uint32 maxAge = 0; uint64 memoryUsed = 0; // Find maxAge, and sum memory used for (uint32 rr=0; rr<=nReads; rr++) { if (maxAge < readAge[rr]) maxAge = readAge[rr]; memoryUsed += readLen[rr]; } // Purge oldest until memory is below watermark while ((memoryLimit < memoryUsed) && (maxAge > 1)) { fprintf(stderr, "purgeReads()-- used "F_U64"MB limit "F_U64"MB -- purge age "F_U32"\n", memoryUsed >> 20, memoryLimit >> 20, maxAge); for (uint32 rr=0; rr<=nReads; rr++) { if (maxAge == readAge[rr]) { memoryUsed -= readLen[rr]; delete [] readSeqFwd[rr]; readSeqFwd[rr] = NULL; //delete [] readSeqRev[rr]; readSeqRev[rr] = NULL; readLen[rr] = 0; readAge[rr] = 0; } } maxAge--; } } canu-1.3/src/overlapInCore/overlapReadCache.H000066400000000000000000000036361272335422300211560ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/overlapInCore/overlapPair-readCache.H * * Modifications by: * * Brian P. Walenz from 2015-JUN-16 to 2015-JUN-23 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include "tgStore.H" class overlapReadCache { public: overlapReadCache(gkStore *gkpStore_, uint64 memLimit); ~overlapReadCache(); private: void loadRead(uint32 id); void loadReads(set reads); void markForLoading(set &reads, uint32 id); public: void loadReads(ovOverlap *ovl, uint32 nOvl); void loadReads(tgTig *tig); void purgeReads(void); char *getRead(uint32 id) { assert(readLen[id] > 0); return(readSeqFwd[id]); }; uint32 getLength(uint32 id) { assert(readLen[id] > 0); return(readLen[id]); }; private: gkStore *gkpStore; uint32 nReads; uint32 *readAge; uint32 *readLen; char **readSeqFwd; //char **readSeqRev; // Save it, or recompute? gkReadData readdata; uint64 memoryLimit; }; canu-1.3/src/pipelines/000077500000000000000000000000001272335422300150655ustar00rootroot00000000000000canu-1.3/src/pipelines/bogart-sweep.pl000066400000000000000000000133051272335422300200230ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2016-MAR-10 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $wrk = "/work/canuassemblies/sent"; my $asm = "test"; system("mkdir -p $wrk") if (! -d $wrk); my $gs = "5000000"; my $b = 6000; my $m = 4; my $t = 1; my $d = "all"; my (@EG, @EB, @EM, @ER, @OL, @RS, @NS, @CS); @EG = ( "0.0100", "0.0200", "0.0300", "0.0400", "0.0500", "0.0600", "0.0700", "0.0800", "0.0900", "0.1000" ); @EB = ( "-0.0050", "+0.0050" ); @EM = ( "-0.0050", "+0.0050" ); @ER = ( "-0.0050", "+0.0050" ); @OL = ( "100", "500", "1000", "2500", "5000", "7500", "10000" ); @RS = ( "-no", "-RS" ); @NS = ( "-no", "-NS" ); @CS = ( "-no", "-CS" ); @EG = ( "0.0400", "0.0500", "0.0600" ); @EB = ( "+0.0125" ); @EM = ( "-0.0125" ); @ER = ( "+0.0000" ); @OL = ( "50", "500", "1000", "2000", "3000", "4000", "5000", "6000", "7000", "8000", "9000", "10000", "11000" ); @RS = ( "-RS" ); @NS = ( "-NS" ); @CS = ( "-CS" ); @EG = ( "0.0500" ); @EB = ( "+0.0125" ); @EM = ( "-0.0125" ); @ER = ( "+0.0000" ); @OL = ( "4100", "4200", "4300", "4400", "4500", "4600", "4700", "4800", "4900", "5100", "5200", "5300", "5400", "5500", "5600", "5700", "5800", "5900" ); @RS = ( "-RS" ); @NS = ( "-NS" ); @CS = ( "-CS" ); undef @OL; for (my $ii=1100; $ii<1200; $ii += 1) { push @OL, $ii; } @OL = ( "1135", "1136", "1137", "1138" ); #-unassembled 2 1000 0.75 0.75 2 -repeatdetect 6 11 15 -threads 1 -D most foreach my $eg (@EG) { foreach my $eb (@EB) { foreach my $em (@EM) { foreach my $er (@ER) { foreach my $ol (@OL) { foreach my $rs (@RS) { foreach my $ns (@NS) { foreach my $cs (@CS) { my ($egl, $ebl, $eml, $erl, $oll) = ($eg, $eb, $em, $er, $ol); $ebl = $egl + $1 if ($eb =~ m/^\+(\d+.\d+)/); $ebl = $egl - $1 if ($eb =~ m/^-(\d+.\d+)/); $eml = $egl + $1 if ($em =~ m/^\+(\d+.\d+)/); $eml = $egl - $1 if ($em =~ m/^-(\d+.\d+)/); $erl = $egl + $1 if ($er =~ m/^\+(\d+.\d+)/); $erl = $egl - $1 if ($er =~ m/^-(\d+.\d+)/); $egl = sprintf("%6.4f", $egl); $ebl = sprintf("%6.4f", $ebl); $eml = sprintf("%6.4f", $eml); $erl = sprintf("%6.4f", $erl); $oll = sprintf("%05d", $ol); my $path = "test-eg$egl-eb$ebl-em$eml-er$erl-ol$oll$rs$ns$cs"; print "$path\n"; system("mkdir -p $path") if (! -d $path); open(F, "> $wrk/$path/bogart.sh") or die "can't open '$wrk/$path/bogart.sh' for writing: $!\n"; print F "#!/bin/sh\n"; print F "\n"; print F "cd $wrk/$path\n"; print F "\n"; print F "if [ ! -e test.tigStore ] ; then\n"; print F " /work/canu/FreeBSD-amd64/bin/bogart \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -O $wrk/$asm.ovlStore \\\n"; print F " -T test.tigStore -o test\\\n"; print F " -B $b -M $m -threads $t \\\n"; print F " -gs $gs \\\n"; print F " -eg $egl -eb $ebl -em $eml -er $erl -el $ol \\\n"; print F " -RS \\\n" if ($rs eq "-RS"); print F " -NS \\\n" if ($ns eq "-NS"); print F " -CS \\\n" if ($cs eq "-CS"); print F " -unassembled 2 1000 0.75 0.75 2 \\\n"; print F " -repeatdetect 6 32 5 \\\n"; print F " -D $d \\\n" if (length($d) > 0); print F " > bogart.err 2>& 1\n"; print F "fi\n"; print F "\n"; close(F); open(F, "> $wrk/$path/utgcns.sh") or die "can't open '$wrk/$path/utgcns.sh' for writing: $!\n"; print F "#!/bin/sh\n"; print F "\n"; print F "cd $wrk/$path\n"; print F "\n"; print F "if [ ! -e test.fasta ] ; then\n"; print F " /work/canu/FreeBSD-amd64/bin/utgcns \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -T test.tigStore 1 . \\\n"; print F " -O test.cns -L test.lay -A test.fasta\n"; print F "fi\n"; print F "\n"; print F "rm -f test.tigStore/seqDB.v002.dat\n"; print F "rm -f test.tigStore/seqDB.v002.tig\n"; print F "\n"; print F "/work/canu/FreeBSD-amd64/bin/tgStoreLoad \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -T test.tigStore 2 \\\n"; print F " test.cns\n"; print F "\n"; print F "/work/canu/FreeBSD-amd64/bin/tgStoreDump \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -T test.tigStore 2 \\\n"; print F " -consensus -fasta -contigs -bubbles \\\n"; print F "> contigs.fasta\n"; print F "\n"; print F "rm -f *.delta\n"; print F "rm -f *.coords\n"; print F "rm -f *.png\n"; print F "\n"; print F "sh /work/scripts/dotplot.sh usmarc /data/references/salmonella_enterica_usmarc_3124.1-cp006631.1.fasta contigs.fasta\n"; print F "sh /work/scripts/dotplot.sh serge /data/references/salmonella_enterica_serge.fasta contigs.fasta\n"; print F "\n"; print F "cp -fp usmarc.png $wrk/$path.usmarc.png\n"; print F "cp -fp serge.png $wrk/$path.serge.png\n"; print F "\n"; close(F); system("sh $wrk/$path/bogart.sh"); system("qsub -q vomit.q -cwd -j y -o /dev/null $wrk/$path/utgcns.sh > /dev/null 2>&1"); } } } } } } } } canu-1.3/src/pipelines/canu.pl000066400000000000000000000427351272335422300163630ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g.pl # # Modifications by: # # Brian P. Walenz from 2015-FEB-27 to 2015-AUG-26 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-NOV-03 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-19 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; use FindBin; use Cwd; use lib "$FindBin::RealBin/lib"; use lib "$FindBin::RealBin/lib/canu/lib/perl5"; use lib "$FindBin::RealBin/lib/canu/lib64/perl5"; use File::Path qw(make_path remove_tree); use Carp; use canu::Defaults; use canu::Execution; use canu::Configure; use canu::Grid; use canu::Grid_SGE; use canu::Grid_Slurm; use canu::Grid_PBSTorque; use canu::Grid_LSF; use canu::Gatekeeper; use canu::Meryl; use canu::OverlapInCore; use canu::OverlapMhap; use canu::OverlapMMap; use canu::OverlapStore; use canu::CorrectReads; use canu::ErrorEstimate; use canu::OverlapBasedTrimming; use canu::OverlapErrorAdjustment; use canu::Unitig; use canu::Consensus; use canu::Output; use canu::HTML; my @specFiles; # Files of specs my @specOpts; # Command line specs my @inputFiles; # Command line inputs, later inputs in spec files are added # Initialize our defaults. Must be done before defaults are reported in printOptions() below. setDefaults(); # The bin directory is needed for -version, can only be set after setDefaults(), but really should be # set after checkParameters() so it can know pathMap. my $bin = getBinDirectory(); # Path to binaries, reset later. my $cmd = undef; # Temporary string passed to system(). my $wrk = undef; # Path to our assembly directory. my $asm = undef; # Name of our assembly. # Check for the presence of a -options switch BEFORE we do any work. # This lets us print the default values of options. foreach my $arg (@ARGV) { if (($arg eq "-options") || ($arg eq "-defaults")) { printOptions(); exit(0); } if (($arg eq "-version") || ($arg eq "--version")) { system("$bin/gatekeeperCreate --version"); exit(0); } } # By default, all three steps are run. Options -correct, -trim and -assemble # can limit the pipeline to just that stage. # At some pain, we stash the original options for later use. We need # to use these when we resubmit ourself to the grid. We can't simply dump # all of @ARGV into here, because we need to fix up relative paths first. my $mode = undef; my $step = "run"; my $haveRaw = 0; my $haveCorrected = 0; while (scalar(@ARGV)) { my $arg = shift @ARGV; if ($arg =~ m/^-d/) { $wrk = shift @ARGV; $wrk = "$ENV{'PWD'}/$wrk" if ($wrk !~ m!^/!); addCommandLineOption("-d \"$wrk\""); } elsif ($arg eq "-p") { $asm = shift @ARGV; addCommandLineOption("-p \"$asm\""); } elsif ($arg eq "-s") { my $spec = shift @ARGV; $spec = "$ENV{'PWD'}/$spec" if ($spec !~ m!^/!); push @specFiles, $spec; addCommandLineOption("-s \"$spec\""); } elsif ($arg eq "-correct") { $mode = $step = "correct"; addCommandLineOption("-correct"); } elsif ($arg eq "-trim") { $mode = $step = "trim"; addCommandLineOption("-trim"); } elsif ($arg eq "-assemble") { $mode = $step = "assemble"; addCommandLineOption("-assemble"); } elsif ($arg eq "-trim-assemble") { $mode = $step = "trim-assemble"; addCommandLineOption("-trim-assemble"); } elsif (($arg eq "-pacbio-raw") || ($arg eq "-pacbio-corrected") || ($arg eq "-nanopore-raw") || ($arg eq "-nanopore-corrected")) { if ($arg =~ m/pacbio/) { setErrorRate(0.025); setGlobal("corErrorRate", "0.30"); setGlobal("cnsMaxCoverage", 40); setGlobal("utgGraphDeviation", 6); } elsif ($arg =~ m/nanopore/) { setErrorRate(0.048); setGlobal("corErrorRate", "0.50"); setGlobal("cnsMaxCoverage", 40); setGlobal("utgGraphDeviation", 6); } $mode = "trim-assemble" if (!defined($mode) && ($arg =~ m/corrected/)); $mode = "run" if (!defined($mode) && ($arg =~ m/raw/)); $haveCorrected = 1 if ($arg =~ m/corrected/); $haveRaw = 1 if ($arg =~ m/raw/); addCommandLineError("ERROR: File '$ARGV[0]' not found.\n") if (! -e $ARGV[0]); while (-e $ARGV[0]) { my $file = shift @ARGV; $file = "$ENV{'PWD'}/$file" if ($file !~ m!^/!); push @inputFiles, "$arg\0$file"; addCommandLineOption("$arg \"$file\""); } } elsif (-e $arg) { addCommandLineError("ERROR: File supplied on command line; use -s, -pacbio-raw, -pacbio-corrected, -nanopore-raw, or -nanopore-corrected.\n"); } elsif ($arg =~ m/=/) { push @specOpts, $arg; addCommandLineOption("\"$arg\""); } else { addCommandLineError("ERROR: Invalid command line option '$arg'. Did you forget quotes around options with spaces?\n"); } } # Fail if both raw and corrected are supplied. addCommandLineError("ERROR: Canu does not currently support mixing raw and corrected sequences.\n") if ($haveRaw && $haveCorrected); # Fail if some obvious things aren't set. addCommandLineError("ERROR: Assembly name prefix not supplied with -p.\n") if (!defined($asm)); addCommandLineError("ERROR: Directory not supplied with -d.\n") if (!defined($wrk)); # If the mode isn't set - which is allowed only if a gkpStore exists somewhere - be a little smart # and figure out which store exists. $mode = "run" if (!defined($mode) && (-d "$wrk/correction/$asm.gkpStore")); $mode = "trim-assemble" if (!defined($mode) && (-d "$wrk/trimming/$asm.gkpStore")); $mode = "assemble" if (!defined($mode) && (-d "$wrk/unitigging/$asm.gkpStore")); # Load paramters from the defaults files @inputFiles = setParametersFromFile("$bin/canu.defaults", @inputFiles) if (-e "$bin/canu.defaults"); @inputFiles = setParametersFromFile("$ENV{'HOME'}/.canu", @inputFiles) if (-e "$ENV{'HOME'}/.canu"); # For each of the spec files, parse it, setting parameters and remembering any input files discovered. foreach my $specFile (@specFiles) { @inputFiles = setParametersFromFile($specFile, @inputFiles); } # Set parameters from the command line. setParametersFromCommandLine(@specOpts); # When resuming a run without input files, set the error rates based on library type in the gkpStore. If the user set error, do nothing # check if we have gkpStores but no input files and reset error rates based on gkpStore if (scalar(@inputFiles) == 0 && ! defined(getGlobal("errorRate"))) { my $gkpStore = undef; $gkpStore = "$wrk/correction/$asm.gkpStore" if -e "$wrk/correction/$asm.gkpStore/libraries.txt"; $gkpStore = "$wrk/trimming/$asm.gkpStore" if -e "$wrk/trimming/$asm.gkpStore/libraries.txt"; $gkpStore = "$wrk/unitigging/$asm.gkpStore" if -e "$wrk/unitigging/$asm.gkpStore/libraries.txt"; # set to the default if we can't find anything if (!defined($gkpStore)) { setErrorRate(0.01); } else { my $numPacBioRaw = 0; my $numPacBioCorrected = 0; my $numNanoporeRaw = 0; my $numNanoporeCorrected = 0; open(L, "< $gkpStore/libraries.txt") or caExit("can't open '$gkpStore/libraries.txt' for reading: $!", undef); while () { $numPacBioRaw++ if (m/pacbio-raw/); $numPacBioCorrected++ if (m/pacbio-corrected/); $numNanoporeRaw++ if (m/nanopore-raw/); $numNanoporeCorrected++ if (m/nanopore-corrected/); } if ($numPacBioRaw > 0 || $numPacBioCorrected > 0) { setErrorRate(0.025); setGlobal("corErrorRate", "0.30"); setGlobal("cnsMaxCoverage", 40); } if ($numNanoporeRaw > 0 || $numNanoporeCorrected > 0) { setErrorRate(0.048); setGlobal("corErrorRate", "0.50"); setGlobal("cnsMaxCoverage", 40); } } } # Finish setting parameters, then reset the bin directory using pathMap. checkParameters(); $bin = getBinDirectory(); # If anything complained (invalid option, missing file, etc) printHelp() will trigger and exit. printHelp(); # Detect grid support. If 'gridEngine' isn't set, the execution methods submitScript() and # submitOrRunParallelJob() will return without submitting, or run locally (respectively). This # means that we can leave the default of 'useGrid' to 'true', and execution will do the right thing # when there isn't a grid. print STDERR "-- Detected ", getNumberOfCPUs(), " CPUs and ", getPhysicalMemorySize(), " gigabytes of memory.\n"; detectSGE(); detectSlurm(); detectPBSTorque(); detectLSF(); # Report if no grid engine found, or if the user has disabled grid support. if (!defined(getGlobal("gridEngine"))) { print STDERR "-- No grid engine detected, grid disabled.\n"; } if ((getGlobal("useGrid") eq "0") && (defined(getGlobal("gridEngine")))) { print STDERR "-- Grid engine disabled per useGrid=false option.\n"; setGlobal("gridEngine", undef); } # Finish setting up the grid. This is done AFTER parameters are set from the command line, to # let the user override any of our defaults. configureSGE(); configureSlurm(); configurePBSTorque(); configureLSF(); configureRemote(); # Based on genomeSize, configure the execution of every component. This needs to be done AFTER the grid is setup! configureAssembler(); # Fail immediately if we run the script on the grid, and the gkpStore directory doesn't exist and # we have no input files. Without this check we'd fail only after being scheduled on the grid. my $cor = (-e "$wrk/correction/$asm.gkpStore") || sequenceFileExists("$wrk/$asm.correctedReads") || (-e "$wrk/$asm.correctedReads.gkp"); my $obt = (-e "$wrk/trimming/$asm.gkpStore") || sequenceFileExists("$wrk/$asm.trimmedReads") || (-e "$wrk/$asm.trimmedReads.gkp"); my $utg = (-e "$wrk/unitigging/$asm.gkpStore"); if (($cor + $obt + $utg == 0) && (scalar(@inputFiles) == 0)) { caExit("no input files specified, and store not already created", undef); } # Check that we were supplied a work directory, and that it exists, or we can create it. caExit("no run directory (-d option) specified", undef) if (!defined($wrk)); make_path("$wrk") if (! -d "$wrk"); make_path("$wrk/canu-logs") if (! -d "$wrk/canu-logs"); make_path("$wrk/canu-scripts") if (! -d "$wrk/canu-scripts"); caExit("run directory (-d option) '$wrk' doesn't exist and couldn't be created", undef) if (! -d $wrk); # This environment variable tells the binaries to log their execution in canu-logs/ $ENV{'CANU_DIRECTORY'} = $wrk; # Report the parameters used. writeLog($wrk); # Submit ourself for grid execution? If not grid enabled, or already running on the grid, this # call just returns. The arg MUST be undef. submitScript($wrk, $asm, undef); # # When doing 'run', this sets options for each stage. # - overlapper 'mhap' for correction, 'ovl' for trimming and assembly. # - consensus 'falconpipe' for correction, 'utgcns' for assembly. No consensus in trimming. # - errorRates 15% for correction and 2% for trimming and assembly. Internally, this is # multiplied by three for obt, ovl, cns, etc. # sub setOptions ($$) { my $mode = shift @_; # E.g,. "run" or "trim-assemble" or just plain ol' "trim" my $step = shift @_; # Step we're setting options for. # Decide if we care about running this step in this mode. I almost applied # De Morgan's Laws to this. I don't think it would have been any clearer. if (($mode eq $step) || ($mode eq "run") || (($mode eq "trim-assemble") && ($step eq "trim")) || (($mode eq "trim-assemble") && ($step eq "assemble"))) { # Do run this. } else { return("don't run this"); } # Create directories for the step, if needed. make_path("$wrk/correction") if ((! -d "$wrk/correction") && ($step eq "correct")); make_path("$wrk/trimming") if ((! -d "$wrk/trimming") && ($step eq "trim")); make_path("$wrk/unitigging") if ((! -d "$wrk/unitigging") && ($step eq "assemble")); # Return that we want to run this step. return($step); } # # Pipeline piece # sub overlap ($$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $ovlType = ($tag eq "utg") ? "normal" : "partial"; if (getGlobal("${tag}overlapper") eq "mhap") { mhapConfigure($wrk, $asm, $tag, $ovlType); mhapPrecomputeCheck($wrk, $asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1); # this also does mhapReAlign mhapCheck($wrk, $asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1); } elsif (getGlobal("${tag}overlapper") eq "minimap") { mmapConfigure($wrk, $asm, $tag, $ovlType); mmapPrecomputeCheck($wrk, $asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1); mmapCheck($wrk, $asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1); } else { overlapConfigure($wrk, $asm, $tag, $ovlType); overlapCheck($wrk, $asm, $tag, $ovlType) foreach (1..getGlobal("canuIterationMax") + 1); } createOverlapStore($wrk, $asm, $tag, getGlobal("ovsMethod")); } # # Begin pipeline # if (getGlobal("canuIteration") > 0) { print STDERR "--\n"; print STDERR "-- This is canu parallel iteration #" . getGlobal("canuIteration") . ", out of a maximum of " . getGlobal("canuIterationMax") . " attempts.\n"; } print STDERR "--\n"; print STDERR "-- Final error rates before starting pipeline:\n"; showErrorRates("-- "); if (setOptions($mode, "correct") eq "correct") { print STDERR "--\n"; print STDERR "--\n"; print STDERR "-- BEGIN CORRECTION\n"; print STDERR "--\n"; gatekeeper($wrk, $asm, "cor", @inputFiles); merylConfigure($wrk, $asm, "cor"); merylCheck($wrk, $asm, "cor") foreach (1..getGlobal("canuIterationMax") + 1); merylProcess($wrk, $asm, "cor"); overlap($wrk, $asm, "cor"); buildCorrectionLayouts($wrk, $asm); generateCorrectedReads($wrk, $asm) foreach (1..getGlobal("canuIterationMax") + 1); dumpCorrectedReads($wrk, $asm); estimateCorrectedError($wrk, $asm, "cor"); buildHTML($wrk, $asm, "cor"); my $correctedReads = sequenceFileExists("$wrk/$asm.correctedReads"); caExit("can't find corrected reads in '$wrk/$asm.correctedReads*'", undef) if (!defined($correctedReads)); undef @inputFiles; push @inputFiles, "-pacbio-corrected\0$correctedReads"; } if (setOptions($mode, "trim") eq "trim") { print STDERR "--\n"; print STDERR "--\n"; print STDERR "-- BEGIN TRIMMING\n"; print STDERR "--\n"; gatekeeper($wrk, $asm, "obt", @inputFiles); merylConfigure($wrk, $asm, "obt"); merylCheck($wrk, $asm, "obt") foreach (1..getGlobal("canuIterationMax") + 1); merylProcess($wrk, $asm, "obt"); overlap($wrk, $asm, "obt"); trimReads ($wrk, $asm); splitReads($wrk, $asm); dumpReads ($wrk, $asm); #summarizeReads($wrk, $asm); buildHTML($wrk, $asm, "obt"); my $trimmedReads = sequenceFileExists("$wrk/$asm.trimmedReads"); caExit("can't find trimmed reads in '$wrk/$asm.trimmedReads*'", undef) if (!defined($trimmedReads)); undef @inputFiles; push @inputFiles, "-pacbio-corrected\0$trimmedReads"; } if (setOptions($mode, "assemble") eq "assemble") { print STDERR "--\n"; print STDERR "--\n"; print STDERR "-- BEGIN ASSEMBLY\n"; print STDERR "--\n"; gatekeeper($wrk, $asm, "utg", @inputFiles); merylConfigure($wrk, $asm, "utg"); merylCheck($wrk, $asm, "utg") foreach (1..getGlobal("canuIterationMax") + 1); merylProcess($wrk, $asm, "utg"); overlap($wrk, $asm, "utg"); #readErrorDetection($wrk, $asm); readErrorDetectionConfigure($wrk, $asm); readErrorDetectionCheck($wrk, $asm) foreach (1..getGlobal("canuIterationMax") + 1); overlapErrorAdjustmentConfigure($wrk, $asm); overlapErrorAdjustmentCheck($wrk, $asm) foreach (1..getGlobal("canuIterationMax") + 1); updateOverlapStore($wrk, $asm); unitig($wrk, $asm); unitigCheck($wrk, $asm) foreach (1..getGlobal("canuIterationMax") + 1); consensusConfigure($wrk, $asm); consensusCheck($wrk, $asm) foreach (1..getGlobal("canuIterationMax") + 1); consensusLoad($wrk, $asm); consensusAnalyze($wrk, $asm); outputGraph($wrk, $asm); outputLayout($wrk, $asm); outputSequence($wrk, $asm); outputSummary($wrk, $asm); } exit(0); canu-1.3/src/pipelines/canu/000077500000000000000000000000001272335422300160135ustar00rootroot00000000000000canu-1.3/src/pipelines/canu/Configure.pm000066400000000000000000000733221272335422300203010ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-NOV-27 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-DEC-02 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Configure; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(displayMemoryValue displayGenomeSize configureAssembler); use strict; use Carp qw(cluck); use Sys::Hostname; use canu::Defaults; # This is called to expand parameter ranges for memory and thread parameters. # Examples of valid ranges: # # no units - 1-4:2 - assumes 'g' in the adjust (if memory) # one unit - 1g-4:2 1-4g:2 1-4:2g - all the others are set to 'g' # two units - 1g-4g:2 1g-4:2g 1-4g:2g - bgn/end are the same, stp uses end # all three - 1g-4g:2g - use as is # # Quirks: 1g-2000m will increment every 1m. # 1g-2000m:1g only adds 1g. # 1g-2048m:1g adds 1 and 2g. sub expandRange ($$) { my $var = shift @_; my $val = shift @_; my @v = split ',', $val; my @r; foreach my $v (@v) { my $bgn; my $bgnu; my $end; my $endu; my $stp; my $stpu; # Decode the range. if ($v =~ m/^(\d+\.{0,1}\d*)([kKmMgGtT]{0,1})$/) { $bgn = $1; $bgnu = $2; $end = $1; $endu = $2; $stp = 1; $stpu = $2; } elsif ($v =~ m/^(\d+\.{0,1}\d*)([kKmMgGtT]{0,1})-(\d+\.{0,1}\d*)([kKmMgGtT]{0,1})$/) { $bgn = $1; $bgnu = $2; $end = $3; $endu = $4; $stp = 1; $stpu = $4; } elsif ($v =~ m/^(\d+\.{0,1}\d*)([kKmMgGtT]{0,1})-(\d+\.{0,1}\d*)([kKmMgGtT]{0,1}):(\d+\.{0,1}\d*)([kKmMgGtT]{0,1})$/) { $bgn = $1; $bgnu = $2; $end = $3; $endu = $4; $stp = $5; $stpu = $6; } else { caExit("can't parse '$var' entry '$v'", undef); } # Undef things that are null. The code that follows this was written assuming undef. $bgnu = undef if ($bgnu eq ""); $endu = undef if ($endu eq ""); $stpu = undef if ($stpu eq ""); # Process the range my $def = defined($bgnu) + defined($endu) + defined($stpu); # If no units, this could be a memory or a thread setting. Don't use units. if ($def == 0) { } # If only one unit specified, set the others to the same. elsif ($def == 1) { if (defined($bgnu)) { $endu = $stpu = $bgnu; } elsif (defined($endu)) { $bgnu = $stpu = $endu; } elsif (defined($stpu)) { $bgnu = $endu = $stpu; } } # If two units specified, set the unset as: # bgn or end unset - set based on the other range # stp unset - set on end if stp $end)); } # Nothing to do if all three are set! elsif ($def == 3) { } my $b = adjustMemoryValue("$bgn$bgnu"); my $e = adjustMemoryValue("$end$endu"); my $s = adjustMemoryValue("$stp$stpu"); for (my $ii=$b; $ii<=$e; $ii += $s) { push @r, $ii; } } #print "$var = "; #foreach my $r (@r) { # print "$r "; #} #print "\n"; return(@r); } # Side effect! This will RESET the $global{} parameters to the computed value. This lets # the rest of canu - in particular, the part that runs the jobs - use the correct value. Without # resetting, I'd be making code changes all over the place to support the values returned. sub getAllowedResources ($$$$) { my $tag = shift @_; # Variant, e.g., "cor", "utg" my $alg = shift @_; # Algorithm, e.g., "mhap", "ovl" my $err = shift @_; # Report of things we can't run. my $all = shift @_; # Report of things we can run. # If no grid, or grid not enabled, everything falls under 'lcoal'. my $class = ((getGlobal("useGrid") ne "0") && (defined(getGlobal("gridEngine")))) ? "grid" : "local"; # Figure out limits. my $maxMemory = getGlobal("maxMemory"); my $maxThreads = getGlobal("maxThreads"); my $taskMemory = getGlobal("${tag}${alg}Memory"); # Algorithm limit, "utgovlMemory", etc. my $taskThreads = getGlobal("${tag}${alg}Threads"); # # The task limits MUST be defined. caExit("${tag}${alg}Memory is not defined", undef) if (!defined($taskMemory)); caExit("${tag}${alg}Threads is not defined", undef) if (!defined($taskThreads)); # If the maximum limits aren't set, default to 'unlimited' (for the grid; we'll effectively filter # by the number of jobs we can fit on the hosts) or to the current hardware limits. $maxMemory = (($class eq "grid") ? 1024 * 1024 : getPhysicalMemorySize()) if (!defined($maxMemory)); # 1 PB memory! $maxThreads = (($class eq "grid") ? 1024 : getNumberOfCPUs()) if (!defined($maxThreads)); # 1 k cores! # Build a list of the available hardware configurations we can run on. If grid, we get this # from the list of previously discovered hosts. If local, it's just this machine. my @gridCor; # Number of cores my @gridMem; # GB's of memory my @gridNum; # Number of nodes if ($class eq "grid") { my @grid = split '\0', getGlobal("availableHosts"); if (scalar(@grid) == 0) { caExit("invalid useGrid (" . getGlobal("useGrid") . ") and gridEngine (" . getGlobal("gridEngine") . "); found no execution hosts - is grid available from this host?", undef); } foreach my $g (@grid) { my ($cpu, $mem, $num) = split '-', $g; push @gridCor, $cpu; push @gridMem, $mem; push @gridNum, $num; } } else { push @gridCor, $maxThreads; push @gridMem, $maxMemory; push @gridNum, 1; } # The task usually has multiple choices, and we have a little optimization problem to solve. For each # pair of memory/threads, compute three things: # a) how many processes we can get running # b) how many cores we can get running # c) how much memory we can consume # We then (typically) want to maximize the number of cores we can get running. # Other options would be number of cores * amount of memory. my @taskMemory = expandRange("${tag}${alg}Memory", $taskMemory); my @taskThreads = expandRange("${tag}${alg}Threads", $taskThreads); # Filter out task settings that can't be run based on the gridMemory/gridThreads or masterMemory/masterThreads setting. # (actually, this just reports those that would be filtered; the actual filtering is inline in the algorithm) my $ignoreM; my $ignoreT; foreach my $m (@taskMemory) { $m = adjustMemoryValue($m); } foreach my $m (@taskMemory) { next if ($m <= $maxMemory); $ignoreM .= "," if (defined($ignoreM)); $ignoreM .= "${m}g"; } foreach my $t (@taskThreads) { next if ($t <= $maxThreads); $ignoreT .= "," if (defined($ignoreT)); $ignoreT .= "$t"; } # Too verbose with long value lists # #if (defined($ignoreM) && defined($ignoreT)) { # $err .= "-- Can't use ${tag}${alg}Memory=$ignoreM and ${tag}${alg}Threads=$ignoreT because of maxMemory=${maxMemory}g and maxThreads=$maxThreads limits.\n"; # #} elsif (defined($ignoreM)) { # $err .= "-- Can't use ${tag}${alg}Memory=$ignoreM because of maxMemory=${maxMemory}g limit.\n"; # #} elsif (defined($ignoreT)) { # $err .= "-- Can't use ${tag}${alg}Threads=$ignoreT because of maxThreads=$maxThreads limit.\n"; #} # Find task memory/thread settings that will maximize the number of cores running. This used # to also compute best as 'cores * memory' but that is better handled by ordering the task # settings parameters. The example below will pick the largest (last) configuration that # maximizes core utilization: # # taskThreads = 4,8,32,64 # taskMemory = 16g,32g,64g my ($bestCores, $bestCoresM, $bestCoresT, $availMemoryMin, $availMemoryMax) = (0, undef, undef, undef, undef); foreach my $m (@taskMemory) { foreach my $t (@taskThreads) { next if ($m > $maxMemory); # Bail if either of the suggest settings are next if ($t > $maxThreads); # larger than the maximum allowed. # Save this memory size. ovsMemory uses a list of possible memory sizes to # pick the smallest one that results in an acceptable number of files. $availMemoryMin = $m if (!defined($availMemoryMin) || ($m < $availMemoryMin)); $availMemoryMax = $m if (!defined($availMemoryMax) || ($availMemoryMax < $m)); # For a job using $m GB memory and $t threads, we can compute how many processes will # fit on each node in our set of available machines. The smaller of the two is then # the number of processes we can run on this node. my $processes = 0; my $cores = 0; my $memory = 0; for (my $ii=0; $ii $maxMemory); caExit("invalid taskThread=$taskThreads; maxThreads=$maxThreads", undef) if ($taskThreads > $maxThreads); # Finally, reset the concurrency (if we're running locally) so we don't swamp our poor workstation. my $concurrent = undef; if ($class eq "local") { my $nc = int($maxThreads / $taskThreads); if (($taskThreads * getGlobal("${tag}${alg}Concurrency") > $maxThreads)) { $err .= "-- Reset concurrency from ", getGlobal("${tag}${alg}Concurrency"), " to $nc.\n"; setGlobal("${tag}${alg}Concurrency", $nc); } if (!defined(getGlobal("${tag}${alg}Concurrency"))) { setGlobal("${tag}${alg}Concurrency", $nc); } $concurrent = getGlobal("${tag}${alg}Concurrency"); } # And report. my $nam; if ($alg eq "bat") { $nam = "bogart (unitigger)"; } elsif ($alg eq "cns") { $nam = "utgcns (consensus"; } elsif ($alg eq "cor") { $nam = "falcon_sense (read correction)"; } elsif ($alg eq "meryl") { $nam = "meryl (k-mer counting)"; } elsif ($alg eq "oea") { $nam = "overlap error adjustment"; } elsif ($alg eq "ovb") { $nam = "overlap store parallel bucketizer"; } elsif ($alg eq "ovs") { $nam = "overlap store parallel sorting"; } elsif ($alg eq "red") { $nam = "read error detection (overlap error adjustment)"; } elsif ($alg eq "mhap") { $nam = "mhap (overlapper)"; } elsif ($alg eq "mmap") { $nam = "minimap (overlapper)"; } elsif ($alg eq "ovl") { $nam = "overlapper"; } else { caFailure("unknown task '$alg' in getAllowedResources().", undef); } $all .= "-- Allowed to"; $all .= " run " . substr(" $concurrent", -3) . " job" . (($concurrent == 1) ? " " : "s") . " concurrently," if (defined($concurrent)); $all .= " run under grid control," if (!defined($concurrent)); $all .= " and use up to " . substr(" $taskThreads", -3) . " compute thread" . (($taskThreads == 1) ? " " : "s"); $all .= " and " . substr(" $taskMemory", -4) . " GB memory for stage '$nam'.\n"; return($err, $all); } # Converts number with units to gigabytes. If no units, gigabytes is assumed. sub adjustMemoryValue ($) { my $val = shift @_; return(undef) if (!defined($val)); return($1) if ($val =~ m/^(\d+\.{0,1}\d*)$/); return($1 / 1024 / 1024) if ($val =~ m/^(\d+\.{0,1}\d*)[kK]$/); return($1 / 1024) if ($val =~ m/^(\d+\.{0,1}\d*)[mM]$/); return($1) if ($val =~ m/^(\d+\.{0,1}\d*)[gG]$/); return($1 * 1024) if ($val =~ m/^(\d+\.{0,1}\d*)[tT]$/); return($1 * 1024 * 1024) if ($val =~ m/^(\d+\.{0,1}\d*)[pP]$/); die "Invalid memory value '$val'\n"; } # Converts gigabytes to number with units. sub displayMemoryValue ($) { my $val = shift @_; return(($val * 1024 * 1024) . "k") if ($val < adjustMemoryValue("1m")); return(($val * 1024) . "m") if ($val < adjustMemoryValue("1g")); return(($val) . "g") if ($val < adjustMemoryValue("1t")); return(($val / 1024) . "t"); } # Converts number with units to bases. sub adjustGenomeSize ($) { my $val = shift @_; return(undef) if (!defined($val)); return($1) if ($val =~ m/^(\d+\.{0,1}\d*)$/i); return($1 * 1000) if ($val =~ m/^(\d+\.{0,1}\d*)[kK]$/i); return($1 * 1000000) if ($val =~ m/^(\d+\.{0,1}\d*)[mM]$/i); return($1 * 1000000000) if ($val =~ m/^(\d+\.{0,1}\d*)[gG]$/i); return($1 * 1000000000000) if ($val =~ m/^(\d+\.{0,1}\d*)[tT]$/i); die "Invalid genome size '$val'\n"; } # Converts bases to number with units. sub displayGenomeSize ($) { my $val = shift @_; return(($val)) if ($val < adjustGenomeSize("1k")); return(($val / 1000) . "k") if ($val < adjustGenomeSize("1m")); return(($val / 1000000) . "m") if ($val < adjustGenomeSize("1g")); return(($val / 1000000000) . "g") if ($val < adjustGenomeSize("1t")); return(($val / 1000000000000) . "t"); } # # If minMemory or minThreads isn't defined, pick a reasonable pair based on genome size. # sub configureAssembler () { # Parse units on things the user possibly set. setGlobal("genomeSize", adjustGenomeSize(getGlobal("genomeSize"))); setGlobal("minMemory", adjustMemoryValue(getGlobal("minMemory"))); setGlobal("maxMemory", adjustMemoryValue(getGlobal("maxMemory"))); # For overlapper and mhap, allow larger maximums for larger genomes. More memory won't help # smaller genomes, and the smaller minimums won't hurt larger genomes (which are probably being # run on larger machines anyway, so the minimums won't be used). # For uncorrected overlapper, both memory and thread count is reduced. Memory because it is # very CPU bound, and thread count because it can be quite unbalanced. if (getGlobal("genomeSize") < adjustGenomeSize("40m")) { setGlobalIfUndef("corOvlMemory", "2-6"); setGlobalIfUndef("corOvlThreads", "1"); setGlobalIfUndef("obtOvlMemory", "4-8"); setGlobalIfUndef("obtOvlThreads", "1-8"); setGlobalIfUndef("utgOvlMemory", "4-8"); setGlobalIfUndef("utgOvlThreads", "1-8"); setGlobalIfUndef("corMhapMemory", "4-6"); setGlobalIfUndef("corMhapThreads", "1-16"); setGlobalIfUndef("obtMhapMemory", "4-6"); setGlobalIfUndef("obtMhapThreads", "1-16"); setGlobalIfUndef("utgMhapMemory", "4-6"); setGlobalIfUndef("utgMhapThreads", "1-16"); setGlobalIfUndef("corMMapMemory", "4-6"); setGlobalIfUndef("corMMapThreads", "1-16"); setGlobalIfUndef("obtMMapMemory", "4-6"); setGlobalIfUndef("obtMMapThreads", "1-16"); setGlobalIfUndef("utgMMapMemory", "4-6"); setGlobalIfUndef("utgMMapThreads", "1-16"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("500m")) { setGlobalIfUndef("corOvlMemory", "2-6"); setGlobalIfUndef("corOvlThreads", "1"); setGlobalIfUndef("obtOvlMemory", "4-8"); setGlobalIfUndef("obtOvlThreads", "1-8"); setGlobalIfUndef("utgOvlMemory", "4-8"); setGlobalIfUndef("utgOvlThreads", "1-8"); setGlobalIfUndef("corMhapMemory", "8-13"); setGlobalIfUndef("corMhapThreads", "1-16"); setGlobalIfUndef("obtMhapMemory", "8-13"); setGlobalIfUndef("obtMhapThreads", "1-16"); setGlobalIfUndef("utgMhapMemory", "8-13"); setGlobalIfUndef("utgMhapThreads", "1-16"); setGlobalIfUndef("corMMapMemory", "8-13"); setGlobalIfUndef("corMMapThreads", "1-16"); setGlobalIfUndef("obtMMapMemory", "8-13"); setGlobalIfUndef("obtMMapThreads", "1-16"); setGlobalIfUndef("utgMMapMemory", "8-13"); setGlobalIfUndef("utgMMapThreads", "1-16"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("2g")) { setGlobalIfUndef("corOvlMemory", "2-8"); setGlobalIfUndef("corOvlThreads", "1"); setGlobalIfUndef("obtOvlMemory", "4-12"); setGlobalIfUndef("obtOvlThreads", "1-8"); setGlobalIfUndef("utgOvlMemory", "4-12"); setGlobalIfUndef("utgOvlThreads", "1-8"); setGlobalIfUndef("corMhapMemory", "16-32"); setGlobalIfUndef("corMhapThreads", "4-16"); setGlobalIfUndef("obtMhapMemory", "16-32"); setGlobalIfUndef("obtMhapThreads", "4-16"); setGlobalIfUndef("utgMhapMemory", "16-32"); setGlobalIfUndef("utgMhapThreads", "4-16"); setGlobalIfUndef("corMMapMemory", "16-32"); setGlobalIfUndef("corMMapThreads", "1-16"); setGlobalIfUndef("obtMMapMemory", "16-32"); setGlobalIfUndef("obtMMapThreads", "1-16"); setGlobalIfUndef("utgMMapMemory", "16-32"); setGlobalIfUndef("utgMMapThreads", "1-16"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("5g")) { setGlobalIfUndef("corOvlMemory", "2-8"); setGlobalIfUndef("corOvlThreads", "1"); setGlobalIfUndef("obtOvlMemory", "4-16"); setGlobalIfUndef("obtOvlThreads", "1-8"); setGlobalIfUndef("utgOvlMemory", "4-16"); setGlobalIfUndef("utgOvlThreads", "1-8"); setGlobalIfUndef("corMhapMemory", "16-48"); setGlobalIfUndef("corMhapThreads", "4-16"); setGlobalIfUndef("obtMhapMemory", "16-48"); setGlobalIfUndef("obtMhapThreads", "4-16"); setGlobalIfUndef("utgMhapMemory", "16-48"); setGlobalIfUndef("utgMhapThreads", "4-16"); setGlobalIfUndef("corMMapMemory", "16-48"); setGlobalIfUndef("corMMapThreads", "1-16"); setGlobalIfUndef("obtMMapMemory", "16-48"); setGlobalIfUndef("obtMMapThreads", "1-16"); setGlobalIfUndef("utgMMapMemory", "16-48"); setGlobalIfUndef("utgMMapThreads", "1-16"); } else { setGlobalIfUndef("corOvlMemory", "2-8"); setGlobalIfUndef("corOvlThreads", "1"); setGlobalIfUndef("obtOvlMemory", "4-16"); setGlobalIfUndef("obtOvlThreads", "1-8"); setGlobalIfUndef("utgOvlMemory", "4-16"); setGlobalIfUndef("utgOvlThreads", "1-8"); setGlobalIfUndef("corMhapMemory", "32-64"); setGlobalIfUndef("corMhapThreads", "4-16"); setGlobalIfUndef("obtMhapMemory", "32-64"); setGlobalIfUndef("obtMhapThreads", "4-16"); setGlobalIfUndef("utgMhapMemory", "32-64"); setGlobalIfUndef("utgMhapThreads", "4-16"); setGlobalIfUndef("corMMapMemory", "32-64"); setGlobalIfUndef("corMMapThreads", "1-16"); setGlobalIfUndef("obtMMapMemory", "32-64"); setGlobalIfUndef("obtMMapThreads", "1-16"); setGlobalIfUndef("utgMMapMemory", "32-64"); setGlobalIfUndef("utgMMapThreads", "1-16"); } # Overlapper block sizes probably don't need to be modified based on genome size. setGlobalIfUndef("corOvlHashBlockLength", 2500000); setGlobalIfUndef("corOvlRefBlockSize", 20000); setGlobalIfUndef("corOvlRefBlockLength", 0); setGlobalIfUndef("obtOvlHashBlockLength", 100000000); setGlobalIfUndef("obtOvlRefBlockSize", 2000000); setGlobalIfUndef("obtOvlRefBlockLength", 0); setGlobalIfUndef("utgOvlHashBlockLength", 100000000); setGlobalIfUndef("utgOvlRefBlockSize", 2000000); setGlobalIfUndef("utgOvlRefBlockLength", 0); # Overlap store construction should be based on the number of overlaps, but we obviously don't # know that until much later. If we set memory too large, we risk (in the parallel version for sure) # inefficiency; too small and we run out of file handles. if (getGlobal("genomeSize") < adjustGenomeSize("300m")) { setGlobalIfUndef("ovsMethod", "sequential"); setGlobalIfUndef("ovbMemory", "2-4"); setGlobalIfUndef("ovbThreads", "1"); setGlobalIfUndef("ovsMemory", "2-8"); setGlobalIfUndef("ovsThreads", "1"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("1g")) { setGlobalIfUndef("ovsMethod", "parallel"); setGlobalIfUndef("ovbMemory", "2-4"); setGlobalIfUndef("ovbThreads", "1"); setGlobalIfUndef("ovsMemory", "4-16"); setGlobalIfUndef("ovsThreads", "1"); } else { setGlobalIfUndef("ovsMethod", "parallel"); setGlobalIfUndef("ovbMemory", "2-4"); setGlobalIfUndef("ovbThreads", "1"); setGlobalIfUndef("ovsMemory", "4-32"); setGlobalIfUndef("ovsThreads", "1"); } # Correction and consensus are somewhat invariant. if (getGlobal("genomeSize") < adjustGenomeSize("40m")) { setGlobalIfUndef("cnsMemory", "8-32"); setGlobalIfUndef("cnsThreads", "1-4"); setGlobalIfUndef("corMemory", "6-16"); setGlobalIfUndef("corThreads", "1-2"); setGlobalIfUndef("cnsPartitions", "8"); setGlobalIfUndef("cnsPartitionMin", "15000"); setGlobalIfUndef("corPartitions", "256"); setGlobalIfUndef("corPartitionMin", "5000"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("1g")) { setGlobalIfUndef("cnsMemory", "16-48"); setGlobalIfUndef("cnsThreads", "2-8"); setGlobalIfUndef("corMemory", "6-20"); setGlobalIfUndef("corThreads", "2-4"); setGlobalIfUndef("cnsPartitions", "64"); setGlobalIfUndef("cnsPartitionMin", "20000"); setGlobalIfUndef("corPartitions", "512"); setGlobalIfUndef("corPartitionMin", "10000"); } else { setGlobalIfUndef("cnsMemory", "64-128"); setGlobalIfUndef("cnsThreads", "2-8"); setGlobalIfUndef("corMemory", "10-32"); setGlobalIfUndef("corThreads", "2-4"); setGlobalIfUndef("cnsPartitions", "256"); setGlobalIfUndef("cnsPartitionMin", "25000"); setGlobalIfUndef("corPartitions", "1024"); setGlobalIfUndef("corPartitionMin", "15000"); } # Meryl too, basically just small or big. This should really be using the number of bases # reported from gatekeeper. if (getGlobal("genomeSize") < adjustGenomeSize("100m")) { setGlobalIfUndef("merylMemory", "4-8"); setGlobalIfUndef("merylThreads", "1-4"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("1g")) { setGlobalIfUndef("merylMemory", "16-64"); setGlobalIfUndef("merylThreads", "1-16"); } else { setGlobalIfUndef("merylMemory", "64-256"); setGlobalIfUndef("merylThreads", "1-32"); } # Overlap error adjustment # # Configuration is primarily done though memory size. This blows up when there are many # short(er) reads and large memory machines are available. # # The limit is arbitrary. # On medicago, with 740,000 reads (median len ~1,500bp), this will result in about 150 jobs. # The memory-only limit generated only 7 jobs. # # On drosophila, with 270,000 reads (median len ~17,000bp), this will result in about 50 jobs. # The memory-only limit generated 36 jobs. # setGlobalIfUndef("redBatchSize", "5000"); setGlobalIfUndef("redBatchLength", ""); setGlobalIfUndef("oeaBatchSize", "25000"); setGlobalIfUndef("oeaBatchLength", ""); if (getGlobal("genomeSize") < adjustGenomeSize("40m")) { setGlobalIfUndef("redMemory", "1-2"); setGlobalIfUndef("redThreads", "1-4"); setGlobalIfUndef("oeaMemory", "1"); setGlobalIfUndef("oeaThreads", "1"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("500m")) { setGlobalIfUndef("redMemory", "2-6"); setGlobalIfUndef("redThreads", "1-6"); setGlobalIfUndef("oeaMemory", "2"); setGlobalIfUndef("oeaThreads", "1"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("2g")) { setGlobalIfUndef("redMemory", "2-8"); setGlobalIfUndef("redThreads", "1-8"); setGlobalIfUndef("oeaMemory", "2"); setGlobalIfUndef("oeaThreads", "1"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("5g")) { setGlobalIfUndef("redMemory", "2-16"); setGlobalIfUndef("redThreads", "1-8"); setGlobalIfUndef("oeaMemory", "4"); setGlobalIfUndef("oeaThreads", "1"); } else { setGlobalIfUndef("redMemory", "2-16"); setGlobalIfUndef("redThreads", "1-8"); setGlobalIfUndef("oeaMemory", "4"); setGlobalIfUndef("oeaThreads", "1"); } # And bogart. if (getGlobal("genomeSize") < adjustGenomeSize("40m")) { setGlobalIfUndef("batMemory", "2-16"); setGlobalIfUndef("batThreads", "1-4"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("500m")) { setGlobalIfUndef("batMemory", "16-64"); setGlobalIfUndef("batThreads", "2-8"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("2g")) { setGlobalIfUndef("batMemory", "32-256"); setGlobalIfUndef("batThreads", "4-16"); } elsif (getGlobal("genomeSize") < adjustGenomeSize("5g")) { setGlobalIfUndef("batMemory", "128-512"); setGlobalIfUndef("batThreads", "8-32"); } else { setGlobalIfUndef("batMemory", "256-1024"); setGlobalIfUndef("batThreads", "16-64"); } # Finally, use all that setup to pick actual values for each component. # # ovsMemory needs to be configured here iff the sequential build method is used. This runs in # the canu process, and needs to have a single memory size. The parallel method will pick a # memory size based on the number of overlaps and submit jobs using that size. my $err; my $all; ($err, $all) = getAllowedResources("", "bat", $err, $all); ($err, $all) = getAllowedResources("cor", "mhap", $err, $all); ($err, $all) = getAllowedResources("obt", "mhap", $err, $all); ($err, $all) = getAllowedResources("utg", "mhap", $err, $all); ($err, $all) = getAllowedResources("", "red", $err, $all); ($err, $all) = getAllowedResources("", "oea", $err, $all); ($err, $all) = getAllowedResources("", "cns", $err, $all); ($err, $all) = getAllowedResources("", "ovb", $err, $all); ($err, $all) = getAllowedResources("", "ovs", $err, $all); ($err, $all) = getAllowedResources("cor", "ovl", $err, $all); ($err, $all) = getAllowedResources("obt", "ovl", $err, $all); ($err, $all) = getAllowedResources("utg", "ovl", $err, $all); ($err, $all) = getAllowedResources("", "meryl", $err, $all); ($err, $all) = getAllowedResources("", "cor", $err, $all); ($err, $all) = getAllowedResources("cor", "mmap", $err, $all); ($err, $all) = getAllowedResources("obt", "mmap", $err, $all); ($err, $all) = getAllowedResources("utg", "mmap", $err, $all); print STDERR "--\n" if (defined($err)); print STDERR $err if (defined($err)); print STDERR "--\n"; print STDERR $all; } canu-1.3/src/pipelines/canu/Consensus.pm000066400000000000000000000332061272335422300203350ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/Consensus.pm # # Modifications by: # # Brian P. Walenz from 2015-MAR-06 to 2015-AUG-25 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-NOV-03 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-DEC-16 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Consensus; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(consensusConfigure consensusCheck consensusLoad consensusAnalyze); use strict; use File::Path qw(make_path remove_tree); use canu::Defaults; use canu::Execution; use canu::Gatekeeper; use canu::Unitig; use canu::HTML; sub computeNumberOfConsensusJobs ($$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $jobs = 0; open(F, "< $wrk/4-unitigger/$asm.partitioningInfo") or caExit("can't open '$wrk/4-unitigger/$asm.partitioningInfo' for reading: $!", undef); while () { if (m/Partition\s+(\d+)\s+has\s+(\d+)\s+unitigs\sand\s+(\d+)\s+fragments./) { $jobs = $1; } } close(F); return($jobs); } sub utgcns ($$$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $jobs = shift @_; #getAllowedResources("", "cns"); open(F, "> $wrk/5-consensus/consensus.sh") or caExit("can't open '$wrk/5-consensus/consensus.sh' for writing: $!", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "if [ \$jobid -gt $jobs ]; then\n"; print F " echo Error: Only $jobs partitions, you asked for \$jobid.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "jobid=`printf %04d \$jobid`\n"; print F "\n"; print F "if [ -e $wrk/5-consensus/\$jobid.cns ] ; then\n"; print F " exit 0\n"; print F "fi\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; print F "\$bin/utgcns \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -T $wrk/$asm.tigStore 1 \$jobid \\\n"; print F " -O $wrk/5-consensus/\$jobid.cns.WORKING \\\n"; #print F " -L $wrk/5-consensus/\$jobid.layout.WORKING \\\n"; #print F " -Q $wrk/5-consensus/\$jobid.fastq.WORKING \\\n"; print F " -maxcoverage " . getGlobal('cnsMaxCoverage') . " \\\n"; print F " -e " . getGlobal("cnsErrorRate") . " \\\n"; print F " -quick \\\n" if (getGlobal("cnsConsensus") eq "quick"); print F " -pbdagcon \\\n" if (getGlobal("cnsConsensus") eq "pbdagcon"); print F " -utgcns \\\n" if (getGlobal("cnsConsensus") eq "utgcns"); print F " -threads " . getGlobal("cnsThreads") . " \\\n"; print F "&& \\\n"; print F "mv $wrk/5-consensus/\$jobid.cns.WORKING $wrk/5-consensus/\$jobid.cns \\\n"; #print F "&& \\\n"; #print F "mv $wrk/5-consensus/\$jobid.layout.WORKING $wrk/5-consensus/\$jobid.layout \\\n"; #print F "&& \\\n"; #print F "mv $wrk/5-consensus/\$jobid.fastq.WORKING $wrk/5-consensus/\$jobid.fastq\n"; print F "\n"; print F "exit 0\n"; close(F); } sub consensusConfigure ($$) { my $WRK = shift @_; # Root work directory my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/5-consensus"; goto allDone if (skipStage($WRK, $asm, "consensusConfigure") == 1); goto allDone if (-e "$wrk/$asm.tigStore/seqDB.v002.tig"); make_path("$path") if (! -d "$path"); # If the gkpStore partitions are older than the tigStore unitig output, assume the unitigs have # changed and remove the gkpStore partition. -M is (annoyingly) 'file age', so we need to # rebuild if gkp is older (larger) than tig. if (-e "$wrk/$asm.gkpStore/partitions/map") { my $gkpTime = -M "$wrk/$asm.gkpStore/partitions/map"; my $tigTime = -M "$wrk/$asm.tigStore/seqDB.v001.tig"; if ($gkpTime > $tigTime) { print STDERR "-- Partitioned gkpStore is older than tigs, rebuild partitioning (gkpStore $gkpTime days old; tigStore $tigTime days old).\n"; if (runCommandSilently($wrk, "rm -rf $wrk/$asm.gkpStore/partitions", 1)) { caExit("failed to remove old partitions ($wrk/$asm.gkpStore/partitions), can't continue until these are removed", undef); } } } # Partition gkpStore if needed. if (! -e "$wrk/$asm.gkpStore/partitions/map") { $cmd = "$bin/gatekeeperPartition \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -P $wrk/4-unitigger/$asm.partitioning \\\n"; $cmd .= "> $path/$asm.partitioned.err 2>&1"; stopBefore("consensusConfigure", $cmd); if (runCommand("$path", $cmd)) { caExit("failed to partition the reads", "$path/$asm.partitioned.err"); } } # Set up the consensus compute. It's in a useless if chain because there used to be # different executables; now they're all rolled into utgcns itself. my $jobs = computeNumberOfConsensusJobs($wrk, $asm); if ((getGlobal("cnsConsensus") eq "quick") || (getGlobal("cnsConsensus") eq "pbdagcon") || (getGlobal("cnsConsensus") eq "utgcns")) { utgcns($wrk, $asm, $jobs); } else { caFailure("unknown consensus style '" . getGlobal("cnsConsensus") . "'", undef); } finishStage: emitStage($WRK, $asm, "consensusConfigure"); buildHTML($WRK, $asm, "utg"); stopAfter("consensusConfigure"); allDone: print STDERR "-- Configured ", computeNumberOfConsensusJobs($wrk, $asm), " consensus jobs.\n"; } # Checks that all consensus jobs are complete, loads them into the store. # sub consensusCheck ($$) { my $WRK = shift @_; # Root work directory my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $attempt = getGlobal("canuIteration"); my $path = "$wrk/5-consensus"; goto allDone if (skipStage($WRK, $asm, "consensusCheck", $attempt) == 1); goto allDone if (-e "$path/cnsjob.files"); goto allDone if (-e "$wrk/$asm.tigStore/seqDB.v002.tig"); # Figure out if all the tasks finished correctly. my $jobs = computeNumberOfConsensusJobs($wrk, $asm); my $currentJobID = "0001"; my @successJobs; my @failedJobs; my $failureMessage = ""; for (my $job=1; $job <= $jobs; $job++) { if (-e "$path/$currentJobID.cns") { push @successJobs, "$path/$currentJobID.cns\n"; } elsif (-e "$path/$currentJobID.cns.gz") { push @successJobs, "$path/$currentJobID.cns.gz\n"; } elsif (-e "$path/$currentJobID.cns.bz2") { push @successJobs, "$path/$currentJobID.cns.bz2\n"; } elsif (-e "$path/$currentJobID.cns.xz") { push @successJobs, "$path/$currentJobID.cns.xz\n"; } else { $failureMessage .= "-- job $path/$currentJobID.cns FAILED.\n"; push @failedJobs, $job; } $currentJobID++; } # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " consensus jobs failed:\n"; print STDERR $failureMessage; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to generate consensus. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- Consensus attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "consensusCheck", $attempt); buildHTML($WRK, $asm, "utg"); submitOrRunParallelJob($WRK, $asm, "cns", $path, "consensus", @failedJobs); return; } finishStage: print STDERR "-- All ", scalar(@successJobs), " consensus jobs finished successfully.\n"; open(L, "> $path/cnsjob.files") or caExit("can't open '$path/cnsjob.files' for writing: $!", undef); print L @successJobs; close(L); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "consensusCheck"); buildHTML($WRK, $asm, "utg"); stopAfter("consensusCheck"); allDone: } sub consensusLoad ($$) { my $WRK = shift @_; # Root work directory my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/5-consensus"; goto allDone if (skipStage($WRK, $asm, "consensusLoad") == 1); goto allDone if (-e "$wrk/$asm.tigStore/seqDB.v002.tig"); # Expects to have a cnsjob.files list of output files from the consensusCheck() function. caExit("can't find '$path/cnsjob.files' for loading tigs into store: $!", undef) if (! -e "$path/cnsjob.files"); # Now just load them. $cmd = "$bin/tgStoreLoad \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -T $wrk/$asm.tigStore 2 \\\n"; $cmd .= " -L $path/cnsjob.files \\\n"; $cmd .= "> $path/cnsjobs.files.tigStoreLoad.err 2>&1"; if (runCommand($path, $cmd)) { caExit("failed to load unitig consensus into tigStore", "$path/cnsjobs.files.tigStoreLoad.err"); } # Remvoe consensus outputs if (-e "$path/cnsjob.files") { print STDERR "-- Purging consensus output after loading to tigStore.\n"; my $Ncns = 0; my $Nfastq = 0; my $Nlayout = 0; my $Nlog = 0; open(F, "< $path/cnsjob.files") or caExit("can't open '$path/cnsjob.files' for reading: $!\n", undef); while () { chomp; if (m/^(.*)\/0*(\d+).cns$/) { my $ID6 = substr("00000" . $2, -6); my $ID4 = substr("000" . $2, -4); my $ID0 = $2; if (-e "$1/$ID4.cns") { $Ncns++; unlink "$1/$ID4.cns"; } if (-e "$1/$ID4.fastq") { $Nfastq++; unlink "$1/$ID4.fastq"; } if (-e "$1/$ID4.layout") { $Nlayout++; unlink "$1/$ID4.layout"; } if (-e "$1/consensus.$ID6.out") { $Nlog++; unlink "$1/consensus.$ID6.out"; } if (-e "$1/consensus.$ID0.out") { $Nlog++; unlink "$1/consensus.$ID0.out"; } } else { caExit("unknown consensus job name '$_'\n", undef); } } close(F); print STDERR "-- Purged $Ncns .cns outputs.\n" if ($Ncns > 0); print STDERR "-- Purged $Nfastq .fastq outputs.\n" if ($Nfastq > 0); print STDERR "-- Purged $Nlayout .layout outputs.\n" if ($Nlayout > 0); print STDERR "-- Purged $Nlog .err log outputs.\n" if ($Nlog > 0); } finishStage: emitStage($WRK, $asm, "consensusLoad"); buildHTML($WRK, $asm, "utg"); stopAfter("consensusLoad"); allDone: reportUnitigSizes($wrk, $asm, 2, "after consenss generation"); } sub consensusAnalyze ($$) { my $WRK = shift @_; # Root work directory my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/5-consensus"; goto allDone if (skipStage($WRK, $asm, "consensusAnalyze") == 1); goto allDone if (-e "$wrk/$asm.tigStore/status.coverageStat"); $cmd = "$bin/tgStoreCoverageStat \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -T $wrk/$asm.tigStore 2 \\\n"; $cmd .= " -s " . getGlobal("genomeSize") . " \\\n"; $cmd .= " -o $wrk/$asm.tigStore.coverageStat \\\n"; $cmd .= "> $wrk/$asm.tigStore.coverageStat.err 2>&1"; if (runCommand($path, $cmd)) { caExit("failed to compute coverage statistics", "$wrk/$asm.tigStore.coverageStat.err"); } unlink "$wrk/$asm.tigStore.coverageStat.err"; finishStage: emitStage($WRK, $asm, "consensusAnalyze"); buildHTML($WRK, $asm, "utg"); touch("$wrk/$asm.tigStore/status.coverageStat"); stopAfter("consensusAnalyze"); allDone: } canu-1.3/src/pipelines/canu/CorrectReads.pm000066400000000000000000001377361272335422300207520ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/CorrectReads.pm # # Modifications by: # # Brian P. Walenz from 2015-APR-09 to 2015-SEP-03 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-19 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-17 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::CorrectReads; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(buildCorrectionLayouts generateCorrectedReads dumpCorrectedReads); use strict; use File::Path qw(make_path remove_tree); use canu::Defaults; use canu::Execution; use canu::Gatekeeper; use canu::HTML; # Returns a coverage: # If $cov not defined, default to desired output coverage * 1.0. # Otherwise, if defined but ends in an 'x', that's desired output coverage * whatever # Otherwise, the coverage is as defined. # sub getCorCov ($$$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $typ = shift @_; my $cov = getGlobal("corMaxEvidenceCoverage$typ"); my $exp = getExpectedCoverage($wrk, $asm); my $des = getGlobal("corOutCoverage"); if (!defined($cov)) { $cov = $des; } elsif ($cov =~ m/(.*)x/) { $cov = int($des * $1); } return($cov); } # Query gkpStore to find the read types involved. Return an error rate that is appropriate for # aligning reads of that type to each other. sub getCorErrorRate ($$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $erate = getGlobal("corErrorRate"); if (defined($erate)) { print STDERR "-- Using overlaps no worse than $erate fraction error for correcting reads (from corErrorRate parameter).\n"; return($erate); } my $numPacBioRaw = 0; my $numPacBioCorrected = 0; my $numNanoporeRaw = 0; my $numNanoporeCorrected = 0; open(L, "< $wrk/$asm.gkpStore/libraries.txt") or caExit("can't open '$wrk/$asm.gkpStore/libraries.txt' for reading: $!", undef); while () { $numPacBioRaw++ if (m/pacbio-raw/); $numPacBioCorrected++ if (m/pacbio-corrected/); $numNanoporeRaw++ if (m/nanopore-raw/); $numNanoporeCorrected++ if (m/nanopore-corrected/); } close(L); $erate = 0.10; # Default; user is stupid and forced correction of corrected reads. $erate = 0.30 if ($numPacBioRaw > 0); $erate = 0.50 if ($numNanoporeRaw > 0); print STDERR "-- Found $numPacBioRaw raw and $numPacBioCorrected corrected PacBio libraries.\n"; print STDERR "-- Found $numNanoporeRaw raw and $numNanoporeCorrected corrected Nanopore libraries.\n"; print STDERR "-- Using overlaps no worse than $erate fraction error for correcting reads.\n"; return($erate); } # Return the number of jobs for 'falcon', 'falconpipe' or 'utgcns' # sub computeNumberOfCorrectionJobs ($$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $nJobs = 0; my $nPerJob = 0; if (getGlobal("corConsensus") eq "falcon" && -e "$wrk/2-correction/correction_inputs" ) { open(F, "ls $wrk/2-correction/correction_inputs/ |") or caExit("can't find list of correction_inputs: $!", undef); while () { $nJobs++ if (m/^\d\d\d\d$/); } close(F); return($nJobs, undef); } if ((getGlobal("corConsensus") eq "utgcns") || (getGlobal("corConsensus") eq "falconpipe") || (getGlobal("corConsensus") eq "falcon")) { my $nPart = getGlobal("corPartitions"); my $nReads = getNumberOfReadsInStore($wrk, $asm); $nPerJob = int($nReads / $nPart + 1); $nPerJob = getGlobal("corPartitionMin") if ($nPerJob < getGlobal("corPartitionMin")); for (my $j=1; $j<=$nReads; $j += $nPerJob) { # We could just divide, except for rounding issues.... $nJobs++; } } return($nJobs, $nPerJob); } # Generate a corStore, dump files for falcon to process, generate a script to run falcon. # sub buildCorrectionLayouts_direct ($$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/2-correction"; # Outer level buildCorrectionLayouts() ensures the task is not finished. my $maxCov = getCorCov($wrk, $asm, "Local"); if (! -e "$wrk/$asm.corStore") { $cmd = "$bin/generateCorrectionLayouts \\\n"; $cmd .= " -rl $path/$asm.readsToCorrect \\\n" if (-e "$path/$asm.readsToCorrect"); $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -O $wrk/$asm.ovlStore \\\n"; $cmd .= " -S $path/$asm.globalScores \\\n" if (-e "$path/$asm.globalScores"); $cmd .= " -T $wrk/$asm.corStore.WORKING \\\n"; $cmd .= " -L " . getGlobal("corMinEvidenceLength") . " \\\n" if (defined(getGlobal("corMinEvidenceLength"))); $cmd .= " -E " . getGlobal("corMaxEvidenceErate") . " \\\n" if (defined(getGlobal("corMaxEvidenceErate"))); $cmd .= " -C $maxCov \\\n" if (defined($maxCov)); $cmd .= " -legacy \\\n" if (defined(getGlobal("corLegacyFilter"))); $cmd .= "> $wrk/$asm.corStore.err 2>&1"; if (runCommand($wrk, $cmd)) { caExit("failed to generate layouts for correction", "$wrk/$asm.corStore.err"); } rename "$wrk/$asm.corStore.WORKING", "$wrk/$asm.corStore"; } # first we call this function to compute partioning my ($jobs, $nPer) = computeNumberOfCorrectionJobs($wrk, $asm); make_path("$path/correction_inputs") if (! -d "$path/correction_inputs"); make_path("$path/correction_outputs") if (! -d "$path/correction_outputs"); if (getGlobal("corConsensus") eq "falcon") { $cmd = "$bin/createFalconSenseInputs \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -T $wrk/$asm.corStore 1 \\\n"; $cmd .= " -o $path/correction_inputs/ \\\n"; $cmd .= " -p " . $jobs . " \\\n"; $cmd .= "> $path/correction_inputs.err 2>&1"; if (runCommand($wrk, $cmd)) { caExit("failed to generate falcon inputs", "$path/correction_inputs.err"); } } if (getGlobal("corConsensus") eq "falcon") { #the second call will confirm we have the proper number of output files and set jobs ($jobs, $nPer) = computeNumberOfCorrectionJobs($wrk, $asm); } #getAllowedResources("", "cor"); open(F, "> $path/correctReads.sh") or caExit("can't open '$path/correctReads.sh'", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "if [ \$jobid -gt $jobs ]; then\n"; print F " echo Error: Only $jobs partitions, you asked for \$jobid.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; if (getGlobal("corConsensus") eq "utgcns") { print F "bgn=`expr \\( \$jobid - 1 \\) \\* $nPer`\n"; print F "end=`expr \\( \$jobid + 0 \\) \\* $nPer`\n"; print F "\n"; } print F "jobid=`printf %04d \$jobid`\n"; print F "\n"; print F "if [ -e \"$path/correction_outputs/\$jobid.fasta\" ] ; then\n"; print F " echo Job finished successfully.\n"; print F " exit 0\n"; print F "fi\n"; print F "\n"; print F "if [ ! -d \"$path/correction_outputs\" ] ; then\n"; print F " mkdir -p \"$path/correction_outputs\"\n"; print F "fi\n"; print F "\n"; print F getBinDirectoryShellCode(); my $erate = getCorErrorRate($wrk, $asm); my $minidt = 1 - $erate; # UTGCNS for correction is writing FASTQ, but needs to write FASTA. The names below were changed to fasta preemptively. if (getGlobal("corConsensus") eq "utgcns") { caExit("UTGCNS for correction is writing FASTQ, but needs to write FASTA", undef); print F "\n"; print F "\$bin/utgcns \\\n"; print F " -u \$bgn-\$end \\\n"; print F " -e $erate \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -T $wrk/$asm.corStore 1 . \\\n"; print F " -O $path/correction_outputs/\$jobid.cns.WORKING \\\n"; print F " -L $path/correction_outputs/\$jobid.layout.WORKING \\\n"; print F " -F $path/correction_outputs/\$jobid.fasta.WORKING \\\n"; print F "&& \\\n"; print F "mv $path/correction_outputs/\$jobid.cns.WORKING $path/correction_outputs/\$jobid.cns \\\n"; print F "&& \\\n"; print F "mv $path/correction_outputs/\$jobid.layout.WORKING $path/correction_outputs/\$jobid.layout \\\n"; print F "&& \\\n"; print F "mv $path/correction_outputs/\$jobid.fasta.WORKING $path/correction_outputs/\$jobid.fasta \\\n"; print F "\n"; } if (getGlobal("corConsensus") eq "falcon") { print F "\n"; print F getGlobal("falconSense") . " \\\n" if ( defined(getGlobal("falconSense"))); print F "\$bin/falcon_sense \\\n" if (!defined(getGlobal("falconSense"))); print F " --min_idt $minidt \\\n"; print F " --min_len " . getGlobal("minReadLength") . "\\\n"; print F " --max_read_len " . 2 * getMaxReadInStore($wrk, $asm) . "\\\n"; print F " --min_ovl_len " . getGlobal("minOverlapLength") . "\\\n"; print F " --min_cov " . getGlobal("corMinCoverage") . " \\\n"; print F " --n_core " . getGlobal("corThreads") . " \\\n"; print F " < $path/correction_inputs/\$jobid \\\n"; print F " > $path/correction_outputs/\$jobid.fasta.WORKING \\\n"; print F " 2> $path/correction_outputs/\$jobid.err \\\n"; print F "&& \\\n"; print F "mv $path/correction_outputs/\$jobid.fasta.WORKING $path/correction_outputs/\$jobid.fasta \\\n"; } close(F); finishStage: ; allDone: } # For falcon_sense, using a pipe and no intermediate files # sub buildCorrectionLayouts_piped ($$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/2-correction"; # Outer level buildCorrectionLayouts() ensures the task is not finished. make_path("$path/correction_inputs") if (! -d "$path/correction_inputs"); make_path("$path/correction_outputs") if (! -d "$path/correction_outputs"); my ($nJobs, $nPerJob) = computeNumberOfCorrectionJobs($wrk, $asm); # Does math based on number of reads and parameters. my $nReads = getNumberOfReadsInStore($wrk, $asm); #getAllowedResources("", "cor"); open(F, "> $path/correctReads.sh") or caExit("can't open '$path/correctReads.sh'", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "if [ \$jobid -gt $nJobs ]; then\n"; print F " echo Error: Only $nJobs partitions, you asked for \$jobid.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; my $bgnID = 1; my $endID = $bgnID + $nPerJob - 1; my $jobID = 1; while ($bgnID < $nReads) { $endID = $bgnID + $nPerJob - 1; $endID = $nReads if ($endID > $nReads); print F "if [ \$jobid -eq $jobID ] ; then\n"; print F " bgn=$bgnID\n"; print F " end=$endID\n"; print F "fi\n"; $bgnID = $endID + 1; $jobID++; } print F "\n"; print F "jobid=`printf %04d \$jobid`\n"; print F "\n"; print F "if [ -e \"$path/correction_outputs/\$jobid.fasta\" ] ; then\n"; print F " echo Job finished successfully.\n"; print F " exit 0\n"; print F "fi\n"; print F "\n"; print F "if [ ! -d \"$path/correction_outputs\" ] ; then\n"; print F " mkdir -p \"$path/correction_outputs\"\n"; print F "fi\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; my $maxCov = getCorCov($wrk, $asm, "Local"); my $erate = getCorErrorRate($wrk, $asm); my $minidt = 1 - $erate; print F "\n"; print F "if [ \"x\$BASH\" != \"x\" ] ; then\n"; # Needs doublequotes, else shell doesn't expand $BASH print F " set -o pipefail\n"; print F "fi\n"; print F "\n"; print F "( \\\n"; print F "\$bin/generateCorrectionLayouts -b \$bgn -e \$end \\\n"; print F " -rl $path/$asm.readsToCorrect \\\n" if (-e "$path/$asm.readsToCorrect"); print F " -G $wrk/$asm.gkpStore \\\n"; print F " -O $wrk/$asm.ovlStore \\\n"; print F " -S $path/$asm.globalScores \\\n" if (-e "$path/$asm.globalScores"); print F " -L " . getGlobal("corMinEvidenceLength") . " \\\n" if (defined(getGlobal("corMinEvidenceLength"))); print F " -E " . getGlobal("corMaxEvidenceErate") . " \\\n" if (defined(getGlobal("corMaxEvidenceErate"))); print F " -C $maxCov \\\n" if (defined($maxCov)); print F " -legacy \\\n" if (defined(getGlobal("corLegacyFilter"))); print F " -F \\\n"; print F "&& \\\n"; print F " touch $path/correction_outputs/\$jobid.dump.success \\\n"; print F ") \\\n"; print F "| \\\n"; print F getGlobal("falconSense") . " \\\n" if ( defined(getGlobal("falconSense"))); print F "\$bin/falcon_sense \\\n" if (!defined(getGlobal("falconSense"))); print F " --min_idt $minidt \\\n"; print F " --min_len " . getGlobal("minReadLength") . "\\\n"; print F " --max_read_len " . 2 * getMaxReadInStore($wrk, $asm) . "\\\n"; print F " --min_ovl_len " . getGlobal("minOverlapLength") . "\\\n"; print F " --min_cov " . getGlobal("corMinCoverage") . " \\\n"; print F " --n_core " . getGlobal("corThreads") . " \\\n"; print F " > $path/correction_outputs/\$jobid.fasta.WORKING \\\n"; print F " 2> $path/correction_outputs/\$jobid.err \\\n"; print F "&& \\\n"; print F "mv $path/correction_outputs/\$jobid.fasta.WORKING $path/correction_outputs/\$jobid.fasta \\\n"; print F "\n"; print F "if [ ! -e \"$path/correction_outputs/\$jobid.dump.success\" ] ; then\n"; print F " echo Read layout generation failed.\n"; print F " mv $path/correction_outputs/\$jobid.fasta $path/correction_outputs/\$jobid.fasta.INCOMPLETE\n"; print F "fi\n"; print F "\n"; print F "exit 0\n"; close(F); finishStage: ; allDone: } sub lengthStats (@) { my @v = sort { $b <=> $a } @_; my $total = 0; my $mean = 0; my $n50 = 0; foreach my $v (@v) { $total += $v; $n50 = $v if ($total < getGlobal("genomeSize") / 2); } $mean = int($total / scalar(@v) + 0.5); return($mean, $n50); } sub quickFilter ($$$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $minTotal = shift @_; my $bin = getBinDirectory(); my $path = "$wrk/2-correction"; my $totCorLengthIn = 0; my $totCorLengthOut = 0; my $minCorLength = 0; open(O, "> $path/$asm.readsToCorrect.WORKING") or caExit("can't open '$path/$asm.readsToCorrect.WORKING' for writing: $!\n", undef); open(F, "$bin/gatekeeperDumpMetaData -G $wrk/$asm.gkpStore -reads | sort -T . -k3nr | ") or caExit("can't dump gatekeeper for read lengths: $!\n", undef); print O "read\toriginalLength\tcorrectedLength\n"; while () { my @v = split '\s+', $_; $totCorLengthIn += $v[2]; $totCorLengthOut += $v[2]; print O "$v[0]\t$v[2]\t0\n"; if ($minTotal != 0 && $totCorLengthIn >= $minTotal) { $minCorLength = $v[2]; last; } } close(F); close(O); rename "$path/$asm.readsToCorrect.WORKING", "$path/$asm.readsToCorrect"; } sub expensiveFilter ($$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/2-correction"; my $maxCov = getCorCov($wrk, $asm, "Local"); if (! -e "$path/$asm.estimate.log") { $cmd = "$bin/generateCorrectionLayouts \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -O $wrk/$asm.ovlStore \\\n"; $cmd .= " -S $path/$asm.globalScores \\\n" if (-e "$path/$asm.globalScores"); $cmd .= " -L " . getGlobal("corMinEvidenceLength") . " \\\n" if (defined(getGlobal("corMinEvidenceLength"))); $cmd .= " -E " . getGlobal("corMaxEvidenceErate") . " \\\n" if (defined(getGlobal("corMaxEvidenceErate"))); $cmd .= " -C $maxCov \\\n" if (defined($maxCov)); $cmd .= " -legacy \\\n" if (defined(getGlobal("corLegacyFilter"))); $cmd .= " -p $path/$asm.estimate"; if (runCommand($wrk, $cmd)) { rename "$path/$asm.estimate.log", "$path/$asm.estimate.log.FAILED"; caExit("failed to generate estimated lengths of corrected reads", "$wrk/$asm.corStore.err"); } } if (runCommandSilently($path, "sort -T . -k4nr -k2nr < $path/$asm.estimate.log > $path/$asm.estimate.correctedLength.log", 1)) { caExit("failed to sort by corrected read length", undef); } if (runCommandSilently($path, "sort -T . -k2nr -k4nr < $path/$asm.estimate.log > $path/$asm.estimate.originalLength.log", 1)) { caExit("failed to sort by original read length", undef); } my $totRawLengthIn = 0; # Bases in raw reads we correct my $totRawLengthOut = 0; # Expected bases in corrected reads my $minRawLength = 0; my $totCorLengthIn = 0; # Bases in raw reads we correct my $totCorLengthOut = 0; # Expected bases in corrected reads my $minCorLength = 0; my $minTotal = getGlobal("genomeSize") * getGlobal("corOutCoverage"); # Lists of reads to correct if we use the raw length or the corrected length as a filter. my %rawReads; my %corReads; my %corReadLen; # The expected length of the corrected reads, based on the filter my @corLengthRawFilter; my @corLengthCorFilter; # Filter! open(F, "< $path/$asm.estimate.originalLength.log"); while () { my @v = split '\s+', $_; next if ($v[0] eq "read"); $corReadLen{$v[0]} = $v[3]; } close(F); open(F, "< $path/$asm.estimate.originalLength.log"); while () { my @v = split '\s+', $_; next if ($v[0] eq "read"); $totRawLengthIn += $v[1]; $totRawLengthOut += $v[3]; #print O "$v[0]\t$v[1]\t$v[3]\n"; $rawReads{$v[0]} = 1; push @corLengthRawFilter, $v[3]; if ($totRawLengthIn >= $minTotal) { # Compare against raw bases $minRawLength = $v[1]; last; } } open(F, "< $path/$asm.estimate.correctedLength.log"); open(O, "| sort -T . -k1n > $path/$asm.readsToCorrect.WORKING") or caExit("can't open sort -k1n > '$path/$asm.readsToCorrect.WORKING' for writing: $!\n", undef); print O "read\toriginalLength\tcorrectedLength\n"; while () { my @v = split '\s+', $_; next if ($v[0] eq "read"); $totCorLengthIn += $v[1]; $totCorLengthOut += $v[3]; print O "$v[0]\t$v[1]\t$v[3]\n"; $corReads{$v[0]} = 1; push @corLengthCorFilter, $v[3]; if ($totCorLengthOut >= $minTotal) { # Compare against corrected bases $minCorLength = $v[3]; last; } } close(O); close(F); rename "$path/$asm.readsToCorrect.WORKING", "$path/$asm.readsToCorrect"; # Generate true/false positive/negative lists. open(F, "< $path/$asm.estimate.correctedLength.log") or die; open(TN, "> $path/$asm.estimate.tn.log") or die; open(FN, "> $path/$asm.estimate.fn.log") or die; open(FP, "> $path/$asm.estimate.fp.log") or die; open(TP, "> $path/$asm.estimate.tp.log") or die; my ($tnReads, $tnBasesR, $tnBasesC, $tnBasesRave, $tnBasesCave) = (0, 0, 0, 0, 0 ); my ($fnReads, $fnBasesR, $fnBasesC, $fnBasesRave, $fnBasesCave) = (0, 0, 0, 0, 0 ); my ($fpReads, $tpBasesR, $tpBasesC, $tpBasesRave, $tpBasesCave) = (0, 0, 0, 0, 0 ); my ($tpReads, $fpBasesR, $fpBasesC, $fpBasesRave, $fpBasesCave) = (0, 0, 0, 0, 0 ); while () { my @v = split '\s+', $_; next if ($v[0] eq "read"); my $er = exists($rawReads{$v[0]}); my $ec = exists($corReads{$v[0]}); if (($er == 0) && ($ec == 0)) { print TN $_; $tnReads++; $tnBasesR += $v[1]; $tnBasesC += $corReadLen{$v[0]}; } # True negative, yay! if (($er == 0) && ($ec == 1)) { print FN $_; $fnReads++; $fnBasesR += $v[1]; $fnBasesC += $corReadLen{$v[0]}; } # False negative. Bad. if (($er == 1) && ($ec == 0)) { print FP $_; $fpReads++; $fpBasesR += $v[1]; $fpBasesC += $corReadLen{$v[0]}; } # False positive. Bad. if (($er == 1) && ($ec == 1)) { print TP $_; $tpReads++; $tpBasesR += $v[1]; $tpBasesC += $corReadLen{$v[0]}; } # True positive, yay! } close(TP); close(FP); close(FN); close(TN); close(F); $tnBasesRave = $tnBasesR / $tnReads if ($tnReads > 0); $tnBasesCave = $tnBasesC / $tnReads if ($tnReads > 0); $fnBasesRave = $fnBasesR / $fnReads if ($fnReads > 0); $fnBasesCave = $fnBasesC / $fnReads if ($fnReads > 0); $fpBasesRave = $fpBasesR / $fpReads if ($fpReads > 0); $fpBasesCave = $fpBasesC / $fpReads if ($fpReads > 0); $tpBasesRave = $tpBasesR / $tpReads if ($tpReads > 0); $tpBasesCave = $tpBasesC / $tpReads if ($tpReads > 0); # Dump a summary of the filter my ($rawFilterMean, $rawFilterN50) = lengthStats(@corLengthRawFilter); my ($corFilterMean, $corFilterN50) = lengthStats(@corLengthCorFilter); open(F, "> $path/$asm.readsToCorrect.summary") or caExit("can't open '$path/$asm.readsToCorrect.summary' for writing: $!\n", undef); print F "Corrected read length filter:\n"; print F "\n"; print F " nReads ", scalar(keys %corReads), "\n"; print F " nBases $totCorLengthIn (input bases)\n"; print F " nBases $totCorLengthOut (corrected bases)\n"; print F " Mean $corFilterMean\n"; print F " N50 $corFilterN50\n"; print F "\n"; print F "Raw read length filter:\n"; print F "\n"; print F " nReads ", scalar(keys %rawReads), "\n"; print F " nBases $totRawLengthIn (input bases)\n"; print F " nBases $totRawLengthOut (corrected bases)\n"; print F " Mean $rawFilterMean\n"; print F " N50 $rawFilterN50\n"; print F "\n"; printf F "TN %9d reads %13d raw bases (%6d ave) %13d corrected bases (%6d ave)\n", $tnReads, $tnBasesR, $tnBasesRave, $tnBasesC, $tnBasesCave; printf F "FN %9d reads %13d raw bases (%6d ave) %13d corrected bases (%6d ave)\n", $fnReads, $fnBasesR, $fnBasesRave, $fnBasesC, $fnBasesCave; printf F "FP %9d reads %13d raw bases (%6d ave) %13d corrected bases (%6d ave)\n", $fpReads, $fpBasesR, $fpBasesRave, $fpBasesC, $fpBasesCave; printf F "TP %9d reads %13d raw bases (%6d ave) %13d corrected bases (%6d ave)\n", $tpReads, $tpBasesR, $tpBasesRave, $tpBasesC, $tpBasesCave; close(F); # Plot a scatter plot of the original vs the expected corrected read lengths. Early versions # also plotted the sorted length vs the other length, but those were not interesting. if (! -e "$path/$asm.estimate.original-x-correctedLength.png") { open(F, "> $path/$asm.estimate.original-x-correctedLength.gp"); print F "set title 'original length (x) vs corrected length (y)'\n"; print F "set xlabel 'original read length'\n"; print F "set ylabel 'corrected read length (expected)'\n"; print F "set pointsize 0.25\n"; print F "\n"; print F "set terminal png size 1024,1024\n"; print F "set output '$path/$asm.estimate.original-x-corrected.lg.png'\n"; print F "plot '$path/$asm.estimate.tn.log' using 2:4 title 'tn', \\\n"; print F " '$path/$asm.estimate.fn.log' using 2:4 title 'fn', \\\n"; print F " '$path/$asm.estimate.fp.log' using 2:4 title 'fp', \\\n"; print F " '$path/$asm.estimate.tp.log' using 2:4 title 'tp'\n"; print F "set terminal png size 256,256\n"; print F "set output '$path/$asm.estimate.original-x-corrected.sm.png'\n"; print F "replot\n"; close(F); if (runCommandSilently($path, "gnuplot < $path/$asm.estimate.original-x-correctedLength.gp > /dev/null 2>&1", 0)) { print STDERR "--\n"; print STDERR "-- WARNING: gnuplot failed; no plots will appear in HTML output.\n"; print STDERR "--\n"; print STDERR "----------------------------------------\n"; } } } sub buildCorrectionLayouts ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/correction"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/2-correction"; # All we do here is decide if the job is finished, and delegate # to the correct function if not finished. # But first we analyze the layouts to pick a length cutoff - or more precisely, to pick a set # of reads to correct. # # If one doesn't want to compute the (expensive) stats, one can just correct more reads. How many more to correct # is dependent on the particular reads. An extra 1x to 5x seems reasonable. # A one-pass algorithm would write layouts to tigStore, computing stats as before, then delete # tigs that shouldn't be corrected. I suspect this will be slower. goto allDone if (skipStage($WRK, $asm, "cor-buildCorrectionLayouts") == 1); goto allDone if (sequenceFileExists("$WRK/$asm.correctedReads")); # Output exists goto allDone if (-e "$path/cnsjob.files"); # Jobs all finished goto allDone if (-e "$path/correctReads.sh"); # Jobs created make_path("$path") if (! -d "$path"); # This will eventually get rolled into overlap store creation. Generate a list of scores for # 'global' overlap filtering. if (! -e "$path/$asm.globalScores") { my $maxCov = getCorCov($wrk, $asm, "Global"); my $minLen = (defined(getGlobal("corMinEvidenceLength"))) ? getGlobal("corMinEvidenceLength") : 0; $cmd = "$bin/filterCorrectionOverlaps \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -O $wrk/$asm.ovlStore \\\n"; $cmd .= " -S $path/$asm.globalScores \\\n"; $cmd .= " -c $maxCov \\\n"; $cmd .= " -l $minLen \\\n"; $cmd .= " -e " . getGlobal("corMaxEvidenceErate") . " \\\n" if (defined(getGlobal("corMaxEvidenceErate"))); $cmd .= " -legacy \\\n" if (defined(getGlobal("corLegacyFilter"))); $cmd .= "> $path/$asm.globalScores.err 2>&1"; if (runCommand($path, $cmd)) { caExit("failed to globally filter overlaps for correction", "$path/$asm.globalScores.err"); } unlink "$path/$asm.globalScores.err"; } # For 'quick' filtering, but more reads to correct, sort the reads by length, and correct the # longest Nx of reads. # # For 'expensive' filtering, but fewer reads to correct, first estimate the corrected lengths # (requires a pass through the overlaps), then pull out the longest Nx of corrected reads. # # Both are required to create a file $asm.readsToCorrect, containing a list of IDs to correct. if (getGlobal("corFilter") eq "quick") { quickFilter($wrk, $asm, (getGlobal("genomeSize") * getGlobal("corOutCoverage"))); } elsif (getGlobal("corFilter") eq "expensive") { expensiveFilter($wrk, $asm); } elsif (getGlobal("corFilter") eq "none" ) { quickFilter($wrk, $asm, 0); } else { caFailure("unknown corFilter '" . getGlobal("corFilter") . "'", undef); } # Set the minimum coverage for a corrected read based on coverage in input reads. if (!defined(getGlobal("corMinCoverage"))) { my $cov = getExpectedCoverage($wrk, $asm); setGlobal("corMinCoverage", 4); setGlobal("corMinCoverage", 4) if ($cov < 60); setGlobal("corMinCoverage", 0) if ($cov <= 20); print STDERR "-- Set corMinCoverage=", getGlobal("corMinCoverage"), " based on read coverage of $cov.\n"; } caExit("failed to create list of reads to correct", undef) if (! -e "$path/$asm.readsToCorrect"); buildCorrectionLayouts_direct($wrk, $asm) if (getGlobal("corConsensus") eq "utgcns"); buildCorrectionLayouts_direct($wrk, $asm) if (getGlobal("corConsensus") eq "falcon"); buildCorrectionLayouts_piped($wrk, $asm) if (getGlobal("corConsensus") eq "falconpipe"); finishStage: emitStage($WRK, $asm, "cor-buildCorrectionLayouts"); buildHTML($WRK, $asm, "cor"); allDone: } sub generateCorrectedReads ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/correction"; # Local work directory my $asm = shift @_; my $attempt = getGlobal("canuIteration"); my $bin = getBinDirectory(); my $path = "$wrk/2-correction"; goto allDone if (skipStage($WRK, $asm, "cor-generateCorrectedReads", $attempt) == 1); goto allDone if (sequenceFileExists("$WRK/$asm.correctedReads")); # Figure out if all the tasks finished correctly. my ($jobs, undef) = computeNumberOfCorrectionJobs($wrk, $asm); my $currentJobID = "0001"; my @successJobs; my @failedJobs; my $failureMessage = ""; for (my $job=1; $job <= $jobs; $job++) { if (-e "$path/correction_outputs/$currentJobID.fasta") { push @successJobs, "$path/correction_outputs/$currentJobID.fasta\n"; } else { $failureMessage .= "-- job $path/correction_outputs/$currentJobID.fasta FAILED.\n"; push @failedJobs, $job; } $currentJobID++; } # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " read correction jobs failed:\n"; print STDERR $failureMessage; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to generate corrected reads. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- generate corrected reads attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "cor-generateCorrectedReads", $attempt); buildHTML($WRK, $asm, "cor"); submitOrRunParallelJob($WRK, $asm, "cor", $path, "correctReads", @failedJobs); return; } finishStage: print STDERR "-- Found ", scalar(@successJobs), " read correction output files.\n"; open(L, "> $path/corjob.files") or caExit("failed to open '$path/corjob.files'", undef); print L @successJobs; close(L); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "cor-generateCorrectedReads"); buildHTML($WRK, $asm, "cor"); stopAfter("readCorrection"); allDone: } sub dumpCorrectedReads ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/correction"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $path = "$wrk/2-correction"; goto allDone if (skipStage($WRK, $asm, "cor-dumpCorrectedReads") == 1); goto allDone if (sequenceFileExists("$WRK/$asm.correctedReads")); print STDERR "-- Concatenating correctReads output.\n"; my $files = 0; my $reads = 0; open(F, "< $path/corjob.files") or caExit("can't open '$path/corjob.files' for reading: $!", undef); open(N, "< $wrk/$asm.gkpStore/readNames.txt") or caExit("can't open '$wrk/$asm.gkpStore/readNames.txt' for reading: $!", undef); open(O, "| gzip -1c > $WRK/$asm.correctedReads.fasta.gz") or caExit("can't open '$WRK/$asm.correctedReads.fasta.gz' for writing: $!", undef); #open(O, "> $WRK/$asm.correctedReads.fasta") or caExit("can't open '$WRK/$asm.correctedReads.fasta' for writing: $!", undef); open(L, "> $WRK/$asm.correctedReads.length") or caExit("can't open '$WRK/$asm.correctedReads.length' for writing: $!", undef); while () { chomp; open(R, "< $_") or caExit("can't open correction output '$_' for reading: $!\n", undef); my $h; # Current header line my $s; # Current sequence my $n; # Next header line my $nameid; # Currently loaded id and name from gkpStore/readNames my $name; $n = ; chomp $n; # Read the first line, the first header. while (!eof(R)) { $h = $n; # Read name. $s = undef; # No sequence yet. $n = ; chomp $n; # Sequence, or the next header. # Read sequence until the next header or we stop reading lines. Perl seems to be # setting EOF when the last line is read, which is early, IMHO. We loop until # we both are EOF and have an empty line. while (($n !~ m/^>/) && ((length($n) > 0) || !eof(R))) { $s .= $n; $n = ; chomp $n; } # Parse the header of the corrected read to find the IID and the split piece number my $rid = undef; # Read ID my $pid = undef; # Piece ID if ($h =~ m/read(\d+)_(\d+)/) { $rid = $1; $pid = $2; } # Load the next line from the gatekeeper ID map file until we find the # correct one. while (!eof(N) && ($rid != $nameid)) { my $rn = ; ($rn =~ m/^(\d+)\s+(.*)$/); $nameid = $1; $name = $2; } # If a match, replace the header with the actual read id. If no match, use the bogus # corrected read name as is. if ($rid eq $nameid) { $h = ">$name iid=${rid}_${pid}"; } # And write the read to the output as FASTA. print O "$h", "\n"; print O "$s", "\n"; # Or as FASTQ #my $q = $s; #$n =~ s/^>/\@/; #$q =~ tr/[A-Z][a-z]/*/; #print O $h, "\n"; #print O $s, "\n"; #print O "+\n"; #print O $q, "\n"; print L "$rid\t$pid\t", length($s), "\n"; $reads++; } $files++; close(R); } close(O); close(F); # Analyze the results. print STDERR "-- Analyzing correctReads output.\n"; my @origLengthHist; my @expcLengthHist; my @corrLengthHist; #my @origExpcDiffLengthHist; #my @origCorrDiffLengthHist; #my @expcCorrDiffLengthHist; my @corrPiecesHist; my $inputReads = 0; my $inputBases = 0; my $failedReads = 0; my $correctedReads = 0; my $correctedBases = 0; my $maxReadLen = 0; my $minDiff = 0; my $maxDiff = 0; open(R, "< $wrk/2-correction/$asm.readsToCorrect") or caExit("can't open '$wrk/2-correction/$asm.readsToCorrect' for reading: $!", undef); open(L, "< $WRK/$asm.correctedReads.length") or caExit("can't open '$WRK./$asm.correctedReads.length' for reading: $!", undef); open(S, "> $wrk/2-correction/$asm.original-expected-corrected-length.dat") or caExit("", undef); my $moreData = 1; my $r = ; chomp $r; # Read first 'read to correct' (input read) my @r = split '\s+', $r; if ($r[0] == 0) { $r = ; chomp $r; # First line was header, read again. @r = split '\s+', $r; } my $l = ; chomp $l; # Read first 'corrected read' (output read) my @l = split '\s+', $l; if ($l[0] == 0) { $l = ; chomp $l; # First line was header, read again. @l = split '\s+', $l; } while ($moreData) { # The 'read to correct' should be a superset of the 'corrected reads'. # There will be only one 'read to correct', but possibly multiple (or even zero) # 'corrected reads'. Read until the IDs differ. For this, we only want to # count the number of pieces and the total length. my $numPieces = 0; my $numBases = 0; while ($l[0] == $r[0]) { $numPieces++; $numBases += $l[2]; last if (eof(L)); # Ugly, break out if we just processed the last 'corrected read'. $l = ; chomp $l; # Read next 'corrected read' @l = split '\s+', $l; } # Add to the length scatter plot (original length, expected length, actual length). print S "$r[0]\t$r[1]\t$r[2]\t$numBases\t", $r[1] - $r[2], "\t", $r[1] - $numBases, "\t", $r[2] - $numBases, "\n"; $minDiff = $r[2] - $numBases if ($r[2] - $numBases < $minDiff); $maxDiff = $r[2] - $numBases if ($r[2] - $numBases > $minDiff); # Add to the histograms. $origLengthHist[$r[1]]++; $expcLengthHist[$r[2]]++; $corrLengthHist[$numBases]++; #$origExpcDiffLengthHist[$r[1] - $r[2]]++; #$origCorrDiffLengthHist[$r[1] - $numBases]++; #$expcCorrDiffLengthHist[$r[2] - $numBases]++; $corrPiecesHist[$numPieces]++; $inputReads++; $inputBases += $r[1]; $failedReads++ if ($numBases == 0); $correctedReads += $numPieces; $correctedBases += $numBases; $maxReadLen = $r[1] if ($maxReadLen < $r[1]); $maxReadLen = $r[2] if ($maxReadLen < $r[2]); $maxReadLen = $numBases if ($maxReadLen < $numBases); # Read next 'read to correct'. If this reads the last line, eof(R) is true, so can't # loop on that. Instead, we set a flag after we process that last line. if (!eof(R)) { $r = ; chomp $r; @r = split '\s+', $r; } else { $moreData = 0; } } close(S); close(R); close(L); # Write a summary of the corrections. open(F, "> $wrk/2-correction/$asm.correction.summary") or caExit("", undef); print F "CORRECTION INPUTS:\n"; print F "-----------------\n"; print F "$inputReads (input reads)\n"; print F "$inputBases (input bases)\n"; print F "\n"; print F "CORRECTION OUTPUTS:\n"; print F "-----------------\n"; print F "$failedReads (reads that failed to generate any corrected bases)\n"; print F "$correctedReads (corrected read pieces)\n"; print F "$correctedBases (corrected bases)\n"; print F "\n"; print F "PIECES PER READ:\n"; print F "---------------\n"; for (my $ii=0; $ii $wrk/2-correction/$asm.originalLength-vs-correctedLength.gp") or caExit("", undef); print F "\n"; print F "set pointsize 0.25\n"; print F "\n"; print F "set title 'original read length vs expected corrected read length'\n"; print F "set xlabel 'original read length'\n"; print F "set ylabel 'expected corrected read length'\n"; print F "\n"; print F "set terminal png size 1024,1024\n"; print F "set output '$wrk/2-correction/$asm.originalLength-vs-expectedLength.lg.png'\n"; print F "plot [0:$maxReadLen] [0:$maxReadLen] '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using 2:3 title 'original (x) vs expected (y)'\n"; print F "set terminal png size 256,256\n"; print F "set output '$wrk/2-correction/$asm.originalLength-vs-expectedLength.sm.png'\n"; print F "replot\n"; print F "\n"; print F "set title 'original read length vs sum of corrected read lengths'\n"; print F "set xlabel 'original read length'\n"; print F "set ylabel 'sum of corrected read lengths'\n"; print F "\n"; print F "set terminal png size 1024,1024\n"; print F "set output '$wrk/2-correction/$asm.originalLength-vs-correctedLength.lg.png'\n"; print F "plot [0:$maxReadLen] [0:$maxReadLen] '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using 2:4 title 'original (x) vs corrected (y)'\n"; print F "set terminal png size 256,256\n"; print F "set output '$wrk/2-correction/$asm.originalLength-vs-correctedLength.sm.png'\n"; print F "replot\n"; print F "\n"; print F "set title 'expected read length vs sum of corrected read lengths'\n"; print F "set xlabel 'expected read length'\n"; print F "set ylabel 'sum of corrected read lengths'\n"; print F "\n"; print F "set terminal png size 1024,1024\n"; print F "set output '$wrk/2-correction/$asm.expectedLength-vs-correctedLength.lg.png'\n"; print F "plot [0:$maxReadLen] [0:$maxReadLen] '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using 3:4 title 'expected (x) vs corrected (y)'\n"; print F "set terminal png size 256,256\n"; print F "set output '$wrk/2-correction/$asm.expectedLength-vs-correctedLength.sm.png'\n"; print F "replot\n"; close(F); if (runCommandSilently("$wrk/2-correction", "gnuplot $wrk/2-correction/$asm.originalLength-vs-correctedLength.gp > /dev/null 2>&1", 0)) { print STDERR "--\n"; print STDERR "-- WARNING: gnuplot failed; no plots will appear in HTML output.\n"; print STDERR "--\n"; print STDERR "----------------------------------------\n"; } # Histograms of lengths, including the difference between expected and actual corrected (the # other two difference plots weren't interesting; original-expected was basically all zero, and # so original-actual was nearly the same as expected-actual. open(F, "> $wrk/2-correction/$asm.length-histograms.gp") or caExit("", undef); print F "set title 'read length'\n"; print F "set ylabel 'number of reads'\n"; print F "set xlabel 'read length, bin width = 250'\n"; print F "\n"; print F "binwidth=250\n"; print F "set boxwidth binwidth\n"; print F "bin(x,width) = width*floor(x/width) + binwidth/2.0\n"; print F "\n"; print F "set terminal png size 1024,1024\n"; print F "set output '$wrk/2-correction/$asm.length-histograms.lg.png'\n"; print F "plot [1:$maxReadLen] [0:] \\\n"; print F " '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using (bin(\$2,binwidth)):(1.0) smooth freq with boxes title 'original', \\\n"; print F " '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using (bin(\$3,binwidth)):(1.0) smooth freq with boxes title 'expected', \\\n"; print F " '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using (bin(\$4,binwidth)):(1.0) smooth freq with boxes title 'corrected'\n"; print F "set terminal png size 256,256\n"; print F "set output '$wrk/2-correction/$asm.length-histograms.sm.png'\n"; print F "replot\n"; print F "\n"; print F "set xlabel 'difference between expected and corrected read length, bin width = 250, min=$minDiff, max=$maxDiff'\n"; print F "\n"; print F "set terminal png size 1024,1024\n"; print F "set output '$wrk/2-correction/$asm.length-difference-histograms.lg.png'\n"; print F "plot [$minDiff:$maxDiff] [0:] \\\n"; print F " '$wrk/2-correction/$asm.original-expected-corrected-length.dat' using (bin(\$7,binwidth)):(1.0) smooth freq with boxes title 'expected - corrected'\n"; print F "set terminal png size 256,256\n"; print F "set output '$wrk/2-correction/$asm.length-difference-histograms.sm.png'\n"; print F "replot\n"; close(F); if (runCommandSilently("$wrk/2-correction", "gnuplot $wrk/2-correction/$asm.length-histograms.gp > /dev/null 2>&1", 0)) { print STDERR "--\n"; print STDERR "-- WARNING: gnuplot failed; no plots will appear in HTML output.\n"; print STDERR "--\n"; print STDERR "----------------------------------------\n"; } # Now that all outputs are (re)written, cleanup the job outputs. print STDERR "--\n"; print STDERR "-- Purging correctReads output after merging to final output file.\n"; print STDERR "-- Purging disabled by saveReadCorrections=true.\n" if (getGlobal("saveReadCorrections") == 1); my $Nsuccess = 0; my $Nerr = 0; my $Nfasta = 0; my $Nlog = 0; if (getGlobal("saveReadCorrections") != 1) { open(F, "< $path/corjob.files") or caExit("can't open '$path/corjob.files' for reading: $!", undef); while () { chomp; if (m/^(.*)\/correction_outputs\/0*(\d+).fasta$/) { my $ID6 = substr("00000" . $2, -6); my $ID4 = substr("000" . $2, -4); my $ID0 = $2; if (-e "$1/correction_outputs/$ID4.dump.success") { $Nsuccess++; unlink "$1/correction_outputs/$ID4.dump.success"; } if (-e "$1/correction_outputs/$ID4.err") { $Nerr++; unlink "$1/correction_outputs/$ID4.err"; } if (-e "$1/correction_outputs/$ID4.fasta") { $Nfasta++; unlink "$1/correction_outputs/$ID4.fasta"; } if (-e "$1/correctReads.$ID6.out") { $Nlog++; unlink "$1/correctReads.$ID6.out"; } if (-e "$1/correctReads.$ID0.out") { $Nlog++; unlink "$1/correctReads.$ID0.out"; } } else { caExit("unknown correctReads job name '$_'\n", undef); } } close(F); } print STDERR "-- Purged $Nsuccess .dump.success sentinels.\n" if ($Nsuccess > 0); print STDERR "-- Purged $Nfasta .fasta outputs.\n" if ($Nfasta > 0); print STDERR "-- Purged $Nerr .err outputs.\n" if ($Nerr > 0); print STDERR "-- Purged $Nlog .out job log outputs.\n" if ($Nlog > 0); finishStage: emitStage($WRK, $asm, "cor-dumpCorrectedReads"); buildHTML($WRK, $asm, "cor"); allDone: print STDERR "--\n"; print STDERR "-- Corrected reads saved in '", sequenceFileExists("$WRK/$asm.correctedReads"), "'.\n"; } canu-1.3/src/pipelines/canu/CorrectReads.txt000066400000000000000000000032031272335422300211320ustar00rootroot00000000000000 buildCorrectionLayouts() ---------------------------------------- filterCorrectionOverlaps (binary) - writes asm.globalScores - log to asm.globalScores.log -- PER READ, #olaps, #scored, #filtered, #saved, reason - log to asm.globalScores.err -- STATS - knows a -S , so write log and stats using that. - params corMaxEvidenceCoverageGlobal - params corMinEvidenceLength ---------------------------------------- quickFilter() or expensiveFilter() Creates asm.readsToCorrect with 'readID', 'originalLength', 'expectedCorrectedLength' quick filter just picks the longest originalLength reads that sum to corOutCoverage * genomeSize no logging, no stats, no plot expensive filter calls generateCorrectionLayouts (binary) to write asm.estimate.log and asm.estimate.stats with expected corrected length based on the overlaps we'd use canu.pl makes asm.estimate.* files with tp/tn rates and a figure canu.pl makes asm.readsToCorrect.summary and asm.estimate.original-x-corrected.png ---------------------------------------- buildCorrectionLayouts (_direct or _piped) generateCorrectionLayouts - reads asm.readsToCorrect (who makes this?) - reads asm.globalScores - writes asm.corStore (direct) - writes falcon-formatted reads for a pipe to compute consensus - params -L corMinEvidenceLength - params -E corMaxEvidenceErate - params -C maxCov (corMaxEvidenceCoverageLocal) - only errors to stderr - writes no log or summary (disabled in canu here, output in the expensiveFilter above) When outputs of the parallel processes are merged, a length file is created. This, with the expensive filter length file, can generate stats on corrections. canu-1.3/src/pipelines/canu/Defaults.pm000066400000000000000000001317151272335422300201300ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/Defaults.pm # # Modifications by: # # Brian P. Walenz from 2015-FEB-27 to 2015-SEP-21 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-21 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-19 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Defaults; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(getCommandLineOptions addCommandLineOption addCommandLineError writeLog caExit caFailure getNumberOfCPUs getPhysicalMemorySize getAllowedResources diskSpace printOptions printHelp setParametersFromFile setParametersFromCommandLine checkParameters getGlobal setGlobal setGlobalIfUndef showErrorRates setErrorRate setDefaults); use strict; use Carp qw(cluck); use Sys::Hostname; use Text::Wrap; use Filesys::Df; # for diskSpace() my %global; my %synops; my %synnam; my $cLineOpts = ""; my $specLog = ""; # Return the second argument, unless the first argument is found in # %global, in which case return that. # sub getGlobal ($) { my $var = shift @_; $var =~ tr/A-Z/a-z/; caFailure("parameter '$var' is not known", undef) if (!exists($global{$var})); return($global{$var}); } sub setGlobalSpecialization ($@) { my $val = shift @_; foreach my $var (@_) { $global{$var} = $val; } return(1); } sub setGlobal ($$) { my $VAR = shift @_; my $var = $VAR; my $val = shift @_; my $set = 0; $var =~ tr/A-Z/a-z/; $val = undef if ($val eq ""); # Set to undefined, the default for many of the options. # Map 'true'/'false' et al. to 0/1. $val = 0 if (($val =~ m/^false$/i) || ($val =~ m/^f$/i)); $val = 1 if (($val =~ m/^true$/i) || ($val =~ m/^t$/i)); # Translate from generic to specialized var foreach my $alg ("ovl", "mhap", "mmap") { foreach my $opt ("gridoptions") { $set += setGlobalSpecialization($val, ("${opt}cor${alg}", "${opt}obt${alg}", "${opt}utg${alg}")) if ($var eq "${opt}${alg}"); } foreach my $opt ("memory", "threads", "concurrency") { $set += setGlobalSpecialization($val, ("cor${alg}${opt}", "obt${alg}${opt}", "utg${alg}${opt}")) if ($var eq "${alg}${opt}"); } } foreach my $opt ("overlapper") { $set += setGlobalSpecialization($val, ("cor${opt}", "obt${opt}", "utg${opt}")) if ($var eq "${opt}"); } # e.g., corOvlHashBlockLength foreach my $opt ("ovlerrorrate", "ovlhashblocklength", "ovlrefblocksize", "ovlrefblocklength", "ovlhashbits", "ovlhashload", "ovlmersize", "ovlmerthreshold", "ovlmerdistinct", "ovlmertotal", "ovlfrequentmers") { $set += setGlobalSpecialization($val, ("cor${opt}", "obt${opt}", "utg${opt}")) if ($var eq "${opt}"); } # e.g., corMhapBlockSize foreach my $opt ("mhapblocksize", "mhapmersize", "mhaprealign", "mhapsensitivity") { $set += setGlobalSpecialization($val, ("cor${opt}", "obt${opt}", "utg${opt}")) if ($var eq "${opt}"); } return if ($set > 0); if ($var eq "errorrate") { setErrorRate($val); return; } # If we got a parameter we don't understand, we should be parsing command line options or # reading spec files, and we can let the usual error handling handle it. addCommandLineError("ERROR: Paramter '$VAR' is not known.\n") if (!exists($global{$var})); $global{$var} = $val; } sub setGlobalIfUndef ($$) { my $var = shift @_; my $val = shift @_; $var =~ tr/A-Z/a-z/; $val = undef if ($val eq ""); # Set to undefined, the default for many of the options. return if (defined($global{$var})); $global{$var} = $val; } sub getCommandLineOptions () { return($cLineOpts); } sub addCommandLineOption ($) { if ($cLineOpts =~ m/\s$/) { $cLineOpts .= "$_[0]"; } else { $cLineOpts .= " $_[0]"; } } sub addCommandLineError($) { $global{'errors'} .= shift @_; } sub writeLog ($) { my $wrk = shift @_; my $time = time(); my $host = hostname(); my $pid = $$; open(F, "> $wrk/canu-logs/${time}_${host}_${pid}_canu"); print F $specLog; close(F); } # Use caExit() for transient errors, like not opening files, processes that die, etc. sub caExit ($$) { my $msg = shift @_; my $log = shift @_; print STDERR "================================================================================\n"; print STDERR "Don't panic, but a mostly harmless error occurred and canu failed.\n"; print STDERR "\n"; # Really should pass in $wrk if (defined($log)) { my $df = diskSpace($log); print STDERR "Disk space available: $df GB\n"; print STDERR "\n"; } if (-e $log) { print STDERR "Last 50 lines of the relevant log file ($log):\n"; print STDERR "\n"; system("tail -n 50 $log"); print STDERR "\n"; } print STDERR "canu failed with '$msg'.\n"; print STDERR "\n"; exit(1); } # Use caFailure() for errors that definitely will require code changes to fix. sub caFailure ($$) { my $msg = shift @_; my $log = shift @_; print STDERR "================================================================================\n"; print STDERR "Please panic. canu failed, and it shouldn't have.\n"; print STDERR "\n"; print STDERR "Stack trace:\n"; print STDERR "\n"; cluck; print STDERR "\n"; if (-e $log) { print STDERR "Last few lines of the relevant log file ($log):\n"; print STDERR "\n"; system("tail -n 50 $log"); } print STDERR "\n"; print STDERR "canu failed with '$msg'.\n"; exit(1); } # # Host management - these really belong in 'Execution.pm' (or 'Utilities.pm') but can't go there # (Execution.pm) and be used here too. # sub getNumberOfCPUs () { my $os = $^O; my $ncpu = 1; # See http://stackoverflow.com/questions/6481005/obtain-the-number-of-cpus-cores-in-linux if ($os eq "freebsd") { $ncpu = int(`/sbin/sysctl -n hw.ncpu`); } if ($os eq "darwin") { $ncpu = int(`/usr/bin/getconf _NPROCESSORS_ONLN`); } if ($os eq "linux" || $os eq "cygwin") { $ncpu = int(`getconf _NPROCESSORS_ONLN`); } return($ncpu); } sub getPhysicalMemorySize () { my $os = $^O; my $memory = 1; if ($os eq "freebsd") { $memory = `/sbin/sysctl -n hw.physmem` / 1024 / 1024 / 1024; } if ($os eq "darwin") { $memory = `/usr/sbin/sysctl -n hw.memsize` / 1024 / 1024 / 1024; } if ($os eq "linux" || $os eq "cygwin") { open(F, "< /proc/meminfo"); # Way to go, Linux! Make it easy on us! while () { if (m/MemTotal:\s+(\d+)/) { $memory = $1 / 1024 / 1024; } } close(F); } return(int($memory + 0.5)); # Poor man's rounding } sub dirname ($) { my $d = shift @_; return($d) if (-d $d); my @d = split '/', $d; pop @d; $d = join('/', @d); return($d); } sub diskSpace ($) { my $wrk = dirname($_[0]); my $df = df($wrk, 1024); my $total = int(10 * $df->{blocks} / 1048576) / 10; my $used = int(10 * $df->{used} / 1048576) / 10; my $free = int(10 * $df->{bfree} / 1048576) / 10; my $avail = int(10 * $df->{bavail} / 1048576) / 10; #print STDERR "Disk space: total $total GB, used $used GB, free $free GB, available $avail GB\n"; return (wantarray) ? ($total, $used, $free, $avail) : $avail; } sub printOptions () { my $pretty = 0; foreach my $k (sort values %synnam) { my $o = $k; my $u = $synops{$k}; next if (length($u) == 0); if ($pretty == 0) { $o = substr("$k ", 0, 40); } else { $Text::Wrap::columns = 100; $o = "$o\n"; $u = wrap(" ", " ", $u) . "\n"; } print "$o$u\n"; } } sub printHelp () { return if (!exists($global{'errors'})); print "\n"; print "usage: canu [-correct | -trim | -assemble] \\\n"; print " [-s ] \\\n"; print " -p \\\n"; print " -d \\\n"; print " genomeSize=[g|m|k] \\\n"; print " errorRate=0.X \\\n"; print " [other-options] \\\n"; print " [-pacbio-raw | -pacbio-corrected | -nanopore-raw | -nanopore-corrected] *fastq\n"; print "\n"; print " By default, all three stages (correct, trim, assemble) are computed.\n"; print " To compute only a single stage, use:\n"; print " -correct - generate corrected reads\n"; print " -trim - generate trimmed reads\n"; print " -assemble - generate an assembly\n"; print "\n"; print " The assembly is computed in the (created) -d , with most\n"; print " files named using the -p .\n"; print "\n"; print " The genome size is your best guess of the genome size of what is being assembled.\n"; print " It is used mostly to compute coverage in reads. Fractional values are allowed: '4.7m'\n"; print " is the same as '4700k' and '4700000'\n"; print "\n"; print " The errorRate is not used correctly (we're working on it). Don't set it\n"; print " If you want to change the defaults, use the various utg*ErrorRate options.\n"; print "\n"; print " A full list of options can be printed with '-options'. All options\n"; print " can be supplied in an optional sepc file.\n"; print "\n"; print " Reads can be either FASTA or FASTQ format, uncompressed, or compressed\n"; print " with gz, bz2 or xz. Reads are specified by the technology they were\n"; print " generated with:\n"; print " -pacbio-raw \n"; print " -pacbio-corrected \n"; print " -nanopore-raw \n"; print " -nanopore-corrected \n"; print "\n"; print "Complete documentation at http://canu.readthedocs.org/en/latest/\n"; print "\n"; print "$global{'errors'}"; print "\n"; exit(1); } sub makeAbsolute ($) { my $var = shift @_; my $val = getGlobal($var); if (defined($val) && ($val !~ m!^/!)) { $val = "$ENV{'PWD'}/$val"; setGlobal($var, $val); $val =~ s/\\\"/\"/g; $val =~ s/\"/\\\"/g; $val =~ s/\\\$/\$/g; $val =~ s/\$/\\\$/g; addCommandLineOption("\"$var=$val\""); } } sub fixCase ($) { my $var = shift @_; my $val = getGlobal($var); if (defined($val)) { $val =~ tr/A-Z/a-z/; setGlobal($var, $val); } } sub setParametersFromFile ($@) { my $specFile = shift @_; my @fragFiles = @_; # Client should be ensuring that the file exists before calling this function. die "specFile '$specFile' not found.\n" if (! -e "$specFile"); $specLog .= "\n"; $specLog .= "###\n"; $specLog .= "### Reading options from '$specFile'\n"; $specLog .= "###\n"; $specLog .= "\n"; open(F, "< $specFile") or caExit("can't open '$specFile' for reading: $!", undef); while () { $specLog .= $_; s/^\s+//; s/\s+$//; next if (m/^\s*\#/); next if (m/^\s*$/); if (-e $_) { my $xx = $_; $xx = "$ENV{'PWD'}/$xx" if ($xx !~ m!^/!); if (-e $xx) { push @fragFiles, $xx; } else { addCommandLineError("ERROR: File not found '$_' after appending absolute path.\n"); } } elsif (m/\s*(\w*)\s*=([^#]*)#*.*$/) { my ($var, $val) = ($1, $2); $var =~ s/^\s+//; $var =~ s/\s+$//; $val =~ s/^\s+//; $val =~ s/\s+$//; undef $val if ($val eq "undef"); setGlobal($var, $val); } else { addCommandLineError("ERROR: File not found or unknown specFile option line '$_'.\n"); } } close(F); return(@fragFiles); } sub setParametersFromCommandLine(@) { my @specOpts = @_; if (scalar(@specOpts) > 0) { $specLog .= "\n"; $specLog .= "###\n"; $specLog .= "### Reading options from the command line.\n"; $specLog .= "###\n"; $specLog .= "\n"; } foreach my $s (@specOpts) { $specLog .= "$s\n"; if ($s =~ m/\s*(\w*)\s*=(.*)/) { my ($var, $val) = ($1, $2); $var =~ s/^\s+//; $var =~ s/\s+$//; $val =~ s/^\s+//; $val =~ s/\s+$//; setGlobal($var, $val); } else { addCommandLineError("ERROR: Misformed command line option '$s'.\n"); } } } sub setExecDefaults ($$) { my $tag = shift @_; my $name = shift @_; $global{"gridOptions${tag}"} = undef; $synops{"gridOptions${tag}"} = "Grid engine options applied to $name jobs"; $global{"${tag}Memory"} = undef; $synops{"${tag}Memory"} = "Amount of memory, in gigabytes, to use for $name jobs"; $global{"${tag}Threads"} = undef; $synops{"${tag}Threads"} = "Number of threads to use for $name jobs"; $global{"${tag}Concurrency"} = undef; $synops{"${tag}Concurrency"} = "If grid not enabled, number of $name jobs to run at the same time; default is n_proc / n_threads"; } sub showErrorRates ($) { my $prefix = shift @_; print STDERR "${prefix}\n"; print STDERR "${prefix}genomeSize -- ", getGlobal("genomeSize"), "\n"; print STDERR "${prefix}errorRate -- ", getGlobal("errorRate"), "\n"; print STDERR "${prefix}\n"; print STDERR "${prefix}corOvlErrorRate -- ", getGlobal("corOvlErrorRate"), "\n"; print STDERR "${prefix}obtOvlErrorRate -- ", getGlobal("obtOvlErrorRate"), "\n"; print STDERR "${prefix}utgOvlErrorRate -- ", getGlobal("utgOvlErrorRate"), "\n"; print STDERR "${prefix}\n"; print STDERR "${prefix}obtErrorRate -- ", getGlobal("obtErrorRate"), "\n"; print STDERR "${prefix}\n"; #print STDERR "${prefix}corErrorRate -- ", getGlobal("corErrorRate"), "\n"; print STDERR "${prefix}cnsErrorRate -- ", getGlobal("cnsErrorRate"), "\n"; } # Defaults are set for yeast: # trimming errorRate = 0.009 obtOvlErrorRate = 0.06 obtErrorRate = 0.035 # assembly errorRate = 0.009 utgOvlErrorRate = 0.06 bogart 0.035 # sub setErrorRate ($@) { my $er = shift @_; my $verbose = shift @_; print STDERR "-- Set errorRate to $er (verbose='$verbose')\n" if (defined($verbose)); # Can NOT call setGlobal() for this, because it calls setErrorRate()!. $global{"errorrate"} = $er; setGlobal("corOvlErrorRate", $er * 3); # Not used, except for realigning setGlobal("obtOvlErrorRate", $er * 3); # Generally must be smaller than utgGraphErrorRate setGlobal("utgOvlErrorRate", $er * 3); setGlobal("obtErrorRate", $er * 3); # Removed, is usually set in CorrectReads, can be set from command line directly. #setGlobal("corErrorRate", $er * 10); # Erorr rate used for raw sequence alignment/consensus setGlobal("cnsErrorRate", $er * 3); showErrorRates("-- ") if (defined($verbose)); } sub setOverlapDefaults ($$$) { my $tag = shift @_; # If 'cor', some parameters are loosened for raw pacbio reads my $name = shift @_; my $default = shift @_; # Sets ${tag}Overlapper # Which overlapper to use. $global{"${tag}Overlapper"} = $default; $synops{"${tag}Overlapper"} = "Which overlap algorithm to use for $name"; # OverlapInCore parameters. $global{"${tag}OvlHashBlockLength"} = undef; $synops{"${tag}OvlHashBlockLength"} = "Amount of sequence (bp) to load into the overlap hash table"; $global{"${tag}OvlRefBlockSize"} = undef; $synops{"${tag}OvlRefBlockSize"} = "Number of reads to search against the hash table per batch"; $global{"${tag}OvlRefBlockLength"} = 0; $synops{"${tag}OvlRefBlockLength"} = "Amount of sequence (bp) to search against the hash table per batch"; $global{"${tag}OvlHashBits"} = ($tag eq "cor") ? 18 : 23; $synops{"${tag}OvlHashBits"} = "Width of the kmer hash. Width 22=1gb, 23=2gb, 24=4gb, 25=8gb. Plus 10b per ${tag}OvlHashBlockLength"; $global{"${tag}OvlHashLoad"} = 0.75; $synops{"${tag}OvlHashLoad"} = "Maximum hash table load. If set too high, table lookups are inefficent; if too low, search overhead dominates run time; default 0.75"; $global{"${tag}OvlMerSize"} = ($tag eq "cor") ? 19 : 22; $synops{"${tag}OvlMerSize"} = "K-mer size for seeds in overlaps"; $global{"${tag}OvlMerThreshold"} = "auto"; $synops{"${tag}OvlMerThreshold"} = "K-mer frequency threshold; mers more frequent than this count are ignored; default 'auto'"; $global{"${tag}OvlMerDistinct"} = undef; $synops{"${tag}OvlMerDistinct"} = "K-mer frequency threshold; the least frequent fraction of distinct mers can seed overlaps"; $global{"${tag}OvlMerTotal"} = undef; $synops{"${tag}OvlMerTotal"} = "K-mer frequency threshold; the least frequent fraction of all mers can seed overlaps"; $global{"${tag}OvlFrequentMers"} = undef; $synops{"${tag}OvlFrequentMers"} = "Do not seed overlaps with these kmers (fasta format)"; # Mhap parameters. $global{"${tag}MhapVersion"} = "2.1"; $synops{"${tag}MhapVersion"} = "Version of the MHAP jar file to use"; $global{"${tag}MhapFilterThreshold"} = "0.000005"; $synops{"${tag}MhapFilterThreshold"} = "Value between 0 and 1. kmers which comprise more than this percentage of the input are downweighted"; $global{"${tag}MhapFilterUnique"} = undef; $synops{"${tag}MhapFilterUnique"} = "Expert option: True or false, supress the low-frequency k-mer distribution based on them being likely noise and not true overlaps. Threshold auto-computed based on error rate and coverage."; $global{"${tag}MhapNoTf"} = undef; $synops{"${tag}MhapNoTf"} = "Expert option: True or false, do not use tf weighting, only idf of tf-idf."; $global{"${tag}MhapBlockSize"} = 3000; $synops{"${tag}MhapBlockSize"} = "Number of reads per 1GB; memory * blockSize = the size of block loaded into memory per job"; $global{"${tag}MhapMerSize"} = ($tag eq "cor") ? 16 : 22; $synops{"${tag}MhapMerSize"} = "K-mer size for seeds in mhap"; $global{"${tag}MhapOrderedMerSize"} = ($tag eq "cor") ? 12 : 22; $synops{"${tag}MhapOrderedMerSize"} = "K-mer size for second-stage filter in mhap"; $global{"${tag}MhapSensitivity"} = undef; $synops{"${tag}MhapSensitivity"} = "Coarse sensitivity level: 'low', 'normal' or 'high'. Usually set automatically based on coverage; 'high' <= 30x < 'normal' < 60x <= 'low'"; $global{"${tag}MMapBlockSize"} = 6000; $synops{"${tag}MMapBlockSize"} = "Number of reads per 1GB; memory * blockSize = the size of block loaded into memory per job"; # minimap parameters. $global{"${tag}MMapMerSize"} = ($tag eq "cor") ? 15 : 22; $synops{"${tag}MMapMerSize"} = "K-mer size for seeds in minmap"; # shared parameters for alignment-free overlappers $global{"${tag}ReAlign"} = undef; $synops{"${tag}ReAlign"} = "Compute actual alignments from overlaps; 'raw' from output, 'final' from overlap store; uses either obtErrorRate or ovlErrorRate, depending on which overlaps are computed"; } sub setDefaults () { ##### General Configuration Options (aka miscellany) $global{"canuIteration"} = 1; # See documentation in Execution.pm $global{"canuIterationMax"} = 2; $global{"showNext"} = undef; $synops{"showNext"} = "Don't run any commands, just report what would run"; $global{"pathMap"} = undef; $synops{"pathMap"} = "File with a hostname to binary directory map"; $global{"shell"} = "/bin/sh"; $synops{"shell"} = "Command interpreter to use; sh-compatible (e.g., bash), NOT C-shell (csh or tcsh); default '/bin/sh'"; $global{"java"} = (exists $ENV{"JAVA_HOME"} && -e "$ENV{'JAVA_HOME'}/bin/java") ? "$ENV{'JAVA_HOME'}/bin/java" : "java"; $synops{"java"} = "Java interpreter to use; at least version 1.8; default 'java'"; ##### Cleanup options $global{"saveOverlaps"} = 0; $synops{"saveOverlaps"} = "Save intermediate overlap files, almost never a good idea"; $global{"saveReadCorrections"} = 0; $synops{"saveReadCorrections"} = "Save intermediate read correction files, almost never a good idea"; $global{"saveMerCounts"} = 0; $synops{"saveMerCounts"} = "Save full mer counting results, sometimes useful"; ##### Error Rates $global{"errorRate"} = undef; $synops{"errorRate"} = "The expected error rate in the corrected reads, typically set based on sequencing type. Set to 0 to try to estimate dynamically. (EXPERIMENTAL)"; $global{"corOvlErrorRate"} = undef; $synops{"corOvlErrorRate"} = "Overlaps above this error rate are not computed"; $global{"obtOvlErrorRate"} = undef; $synops{"obtOvlErrorRate"} = "Overlaps at or below this error rate are used to trim reads"; $global{"utgOvlErrorRate"} = undef; $synops{"utgOvlErrorRate"} = "Overlaps at or below this error rate are used to trim reads"; #$global{"utgErrorRate"} = undef; #$synops{"utgErrorRate"} = "Overlaps at or below this error rate are used to construct unitigs (BOG and UTG)"; $global{"utgGraphDeviation"} = 5; $synops{"utgGraphDeviation"} = "Overlaps this much above median will not be used for initial graph construction (BOGART)"; $global{"utgRepeatDeviation"} = 3; $synops{"utgRepeatDeviation"} = "Overlaps this much above mean unitig error rate will not be used for repeat splitting (BOGART)"; $global{"utgRepeatConfusedBP"} = 5000; $synops{"utgRepeatConfusedBP"} = "Repeats where the next best edge is at least this many bp shorter will not be split (BOGART)"; $global{"corErrorRate"} = undef; $synops{"corErrorRate"} = "Only use raw alignments below this error rate to construct corrected reads"; $global{"cnsErrorRate"} = undef; $synops{"cnsErrorRate"} = "Consensus expects alignments at about this error rate"; ##### Minimums and maximums $global{"minReadLength"} = 1000; $synops{"minReadLength"} = "Reads shorter than this length are not loaded into the assembler; default 1000"; $global{"minOverlapLength"} = 500; $synops{"minOverlapLength"} = "Overlaps shorter than this length are not computed; default 500"; $global{"minMemory"} = undef; $synops{"minMemory"} = "Minimum amount of memory needed to compute the assembly (do not set unless prompted!)"; $global{"minThreads"} = undef; $synops{"minThreads"} = "Minimum number of compute threads suggested to compute the assembly"; $global{"maxMemory"} = undef; $synops{"maxMemory"} = "Maximum memory to use by any component of the assembler"; $global{"maxThreads"} = undef; $synops{"maxThreads"} = "Maximum number of compute threads to use by any component of the assembler"; ##### Stopping conditions $global{"stopOnReadQuality"} = 1; $synops{"stopOnReadQuality"} = "Stop if a significant portion of the input data is too short or has quality value or base composition errors"; $global{"stopBefore"} = undef; $synops{"stopBefore"} = "Tell canu when to halt execution"; $global{"stopAfter"} = undef; $synops{"stopAfter"} = "Tell canu when to halt execution"; ##### Grid Engine configuration, internal parameters. These are filled out in canu.pl, right after this function returns. $global{"availableHosts"} = undef; # Internal list of cpus-memory-nodes describing the grid $global{"gridEngine"} = undef; $global{"gridEngineSubmitCommand"} = undef; $global{"gridEngineHoldOption"} = undef; $global{"gridEngineHoldOptionNoArray"} = undef; $global{"gridEngineSyncOption"} = undef; $global{"gridEngineNameOption"} = undef; $global{"gridEngineArrayOption"} = undef; $global{"gridEngineArrayName"} = undef; $global{"gridEngineArrayMaxJobs"} = undef; $global{"gridEngineOutputOption"} = undef; $global{"gridEnginePropagateCommand"} = undef; $global{"gridEngineThreadsOption"} = undef; $global{"gridEngineMemoryOption"} = undef; $global{"gridEngineMemoryUnits"} = undef; $global{"gridEngineNameToJobIDCommand"} = undef; $global{"gridEngineNameToJobIDCommandNoArray"} = undef; $global{"gridEngineTaskID"} = undef; $global{"gridEngineArraySubmitID"} = undef; $global{"gridEngineJobID"} = undef; ##### Grid Engine Pipeline $global{"useGrid"} = 1; $synops{"useGrid"} = "If 'true', enable grid-based execution; if 'false', run all jobs on the local machine; if 'remote', create jobs for grid execution but do not submit; default 'true'"; foreach my $c (qw(BAT CNS COR MERYL CORMHAP CORMMAP COROVL OBTMHAP OBTOVL OEA OVB OVS RED UTGMHAP UTGMMAP UTGOVL)) { $global{"useGrid$c"} = 1; $synops{"useGrid$c"} = "If 'true', run module $c under grid control; if 'false' run locally."; } ##### Grid Engine configuration, for each step of the pipeline $global{"gridOptions"} = undef; $synops{"gridOptions"} = "Grid engine options applied to all jobs"; $global{"gridOptionsExecutive"} = undef; $synops{"gridOptionsExecutive"} = "Grid engine options applied to the canu executive script"; $global{"gridOptionsJobName"} = undef; $synops{"gridOptionsJobName"} = "Grid jobs job-name suffix"; ##### Grid Engine configuration and parameters, for each step of the pipeline (memory, threads) setExecDefaults("meryl", "mer counting"); setExecDefaults("cor", "read correction"); setExecDefaults("corovl", "overlaps for correction"); setExecDefaults("obtovl", "overlaps for trimming"); setExecDefaults("utgovl", "overlaps for unitig construction"); setExecDefaults("cormhap", "mhap overlaps for correction"); setExecDefaults("obtmhap", "mhap overlaps for trimming"); setExecDefaults("utgmhap", "mhap overlaps for unitig construction"); setExecDefaults("cormmap", "mmap overlaps for correction"); setExecDefaults("obtmmap", "mmap overlaps for trimming"); setExecDefaults("utgmmap", "mmap overlaps for unitig construction"); setExecDefaults("ovb", "overlap store bucketizing"); setExecDefaults("ovs", "overlap store sorting"); setExecDefaults("red", "read error detection"); setExecDefaults("oea", "overlap error adjustment"); setExecDefaults("bat", "unitig construction"); setExecDefaults("cns", "unitig consensus"); ##### Overlapper setOverlapDefaults("cor", "correction", "mhap"); # Overlaps computed for correction setOverlapDefaults("obt", "overlap based trimming", "ovl"); # Overlaps computed for trimming setOverlapDefaults("utg", "unitig construction", "ovl"); # Overlaps computed for unitigging ##### Overlap Store $global{"ovsMethod"} = undef; $synops{"ovsMethod"} = "Use the 'sequential' or 'parallel' algorithm for constructing an overlap store; default 'sequential'"; ##### Mers $global{"merylMemory"} = undef; $synops{"merylMemory"} = "Amount of memory, in gigabytes, to use for mer counting"; $global{"merylThreads"} = undef; $synops{"merylThreads"} = "Number of threads to use for mer counting"; $global{"merylConcurrency"} = undef; $synops{"merylConcurrency"} = "Unused, there is only one process"; ##### Overlap Based Trimming $global{"obtErrorRate"} = undef; $synops{"obtErrorRate"} = "Stringency of overlaps to use for trimming"; $global{"trimReadsOverlap"} = 1; $synops{"trimReadsOverlap"} = "Minimum overlap between evidence to make contiguous trim; default '1'"; $global{"trimReadsCoverage"} = 1; $synops{"trimReadsCoverage"} = "Minimum depth of evidence to retain bases; default '1'"; #$global{"splitReads..."} = 1; #$synops{"splitReads..."} = ""; ##### Fragment/Overlap Error Correction $global{"enableOEA"} = 1; $synops{"enableOEA"} = "Do overlap error adjustment - comprises two steps: read error detection (RED) and overlap error adjustment (OEA); default 'true'"; $global{"redBatchSize"} = undef; $synops{"redBatchSize"} = "Number of reads per fragment error detection batch"; $global{"redBatchLength"} = undef; $synops{"redBatchLength"} = "Number of bases per fragment error detection batch"; $global{"oeaBatchSize"} = undef; $synops{"oeaBatchSize"} = "Number of reads per overlap error correction batch"; $global{"oeaBatchLength"} = undef; $synops{"oeaBatchLength"} = "Number of bases per overlap error correction batch"; ##### Unitigger & BOG & bogart Options $global{"unitigger"} = "bogart"; $synops{"unitigger"} = "Which unitig algorithm to use; only 'bogart' supported; default 'bogart'"; $global{"genomeSize"} = undef; $synops{"genomeSize"} = "An estimate of the size of the genome"; $global{"batOptions"} = undef; $synops{"batOptions"} = "Advanced options to bogart"; $global{"batMemory"} = undef; $synops{"batMemory"} = "Approximate maximum memory usage, in gigabytes, default is the maxMemory limit"; $global{"batThreads"} = undef; $synops{"batThreads"} = "Number of threads to use; default is the maxThreads limit"; $global{"batConcurrency"} = undef; $synops{"batConcurrency"} = "Unused, only one process supported"; ##### Unitig Filtering Options (also set in bogart/bogart.C) $global{"contigFilter"} = "2 1000 0.75 0.75 2"; $synops{"contigFilter"} = "Parameters to filter out 'unassembled' unitigs: minReads; minLength; singleReadSpan; lowCovSpan, lowCovDepth"; ##### Consensus Options $global{"cnsPartitions"} = undef; $synops{"cnsPartitions"} = "Partition consensus into N jobs"; $global{"cnsPartitionMin"} = undef; $synops{"cnsPartitionMin"} = "Don't make a consensus partition with fewer than N reads"; $global{"cnsMaxCoverage"} = 0; $synops{"cnsMaxCoverage"} = "Limit unitig consensus to at most this coverage; default '0' = unlimited"; $global{"cnsConsensus"} = "pbdagcon"; $synops{"cnsConsensus"} = "Which consensus algorithm to use; 'pbdagcon' (fast, reliable); 'utgcns' (multialignment output); 'quick' (single read mosaic); default 'pbdagcon'"; ##### Correction Options $global{"corPartitions"} = undef; $synops{"corPartitions"} = "Partition read correction into N jobs"; $global{"corPartitionMin"} = undef; $synops{"corPartitionMin"} = "Don't make a read correction partition with fewer than N reads"; $global{"corMinEvidenceLength"} = undef; $synops{"corMinEvidenceLength"} = "Limit read correction to only overlaps longer than this; default: unlimited"; $global{"corMaxEvidenceErate"} = undef; $synops{"corMaxEvidenceErate"} = "Limit read correction to only overlaps at or below this fraction error; default: unlimited"; $global{"corMaxEvidenceCoverageGlobal"}= "1.0x"; $synops{"corMaxEvidenceCoverageGlobal"}= "Limit reads used for correction to supporting at most this coverage; default: '1.0x' = 1.0 * estimated coverage"; $global{"corMaxEvidenceCoverageLocal"} = "2.0x"; $synops{"corMaxEvidenceCoverageLocal"} = "Limit reads being corrected to at most this much evidence coverage; default: '2.0x' = 2.0 * estimated coverage"; $global{"corOutCoverage"} = 40; $synops{"corOutCoverage"} = "Only correct the longest reads up to this coverage; default 40"; $global{"corMinCoverage"} = undef; $synops{"corMinCoverage"} = "Minimum number of bases supporting each corrected base, if less than this sequences are split; default based on input read coverage: 0 <= 30x < 4 < 60x <= 4"; $global{"corFilter"} = "expensive"; $synops{"corFilter"} = "Method to filter short reads from correction; 'quick' or 'expensive'; default 'expensive'"; $global{"corConsensus"} = "falconpipe"; $synops{"corConsensus"} = "Which consensus algorithm to use; only 'falcon' and 'falconpipe' are supported; default 'falconpipe'"; $global{"falconSense"} = undef; $synops{"falconSense"} = "Path to fc_consensus.py or falcon_sense.bin"; $global{"corLegacyFilter"} = undef; $synops{"corLegacyFilter"} = "Expert option: global filter, length * identity (default) or length with broken by identity (if on)"; # Convert all the keys to lowercase, and remember the case-sensitive version foreach my $k (keys %global) { (my $l = $k) =~ tr/A-Z/a-z/; if (! exists($synnam{$l})) { $synnam{$l} = $k; if (!exists($global{$l})) { $global{$l} = $global{$k}; delete $global{$k}; } #print "$k -> $l\n"; } } # If this is set, it breaks the consensus.sh and overlap.sh scripts. Good grief! Why # are you running this in a task array!? if (exists($ENV{getGlobal("gridEngineTaskID")})) { undef $ENV{getGlobal("gridEngineTaskID")}; print STDERR "ENV: ", getGlobal("gridEngineTaskID"), " needs to be unset, done.\n"; } } sub checkParameters () { # # Fiddle with filenames to make them absolute paths. # makeAbsolute("pathMap"); makeAbsolute("corOvlFrequentMers"); makeAbsolute("obtOvlFrequentMers"); makeAbsolute("utgOvlFrequentMers"); # # Adjust case on some of them # fixCase("corOverlapper"); fixCase("obtOverlapper"); fixCase("utgOverlapper"); fixCase("corConsensus"); fixCase("cnsConsensus"); fixCase("corFilter"); fixCase("unitigger"); fixCase("stopBefore"); fixCase("stopAfter"); # # Check for inconsistent parameters # if (getGlobal("minReadLength") < getGlobal("minOverlapLength")) { my $mr = getGlobal("minReadLength"); my $mo = getGlobal("minOverlapLength"); addCommandLineError("ERROR: minReadLength=$mr must be at least minOverlapLength=$mo.\n"); # Or we can just reset one or the other.... #print STDERR "-- WARNING: minReadLength reset from $mr to $mo (limited by minOverlapLength)\n"; #setGlobal("minOverlapLength", $mo); } # # Check for invalid usage # foreach my $tag ("cor", "obt", "utg") { if ((getGlobal("${tag}Overlapper") ne "mhap") && (getGlobal("${tag}Overlapper") ne "ovl") && (getGlobal("${tag}Overlapper") ne "minimap")) { addCommandLineError("ERROR: Invalid '${tag}Overlapper' specified (" . getGlobal("${tag}Overlapper") . "); must be 'mhap', 'ovl', or 'minimap'\n"); } } foreach my $tag ("cor", "obt", "utg") { if (getGlobal("${tag}MhapSensitivity") eq "fast") { print STDERR "WARNING: deprecated ${tag}NhapSensitivity=fast replaced with ${tag}MhapSensitivity=low\n"; } if ((getGlobal("${tag}MhapSensitivity") ne undef) && (getGlobal("${tag}MhapSensitivity") ne "low") && (getGlobal("${tag}MhapSensitivity") ne "normal") && (getGlobal("${tag}MhapSensitivity") ne "high")) { addCommandLineError("ERROR: Invalid '${tag}MhapSensitivity' specified (" . getGlobal("${tag}MhapSensitivity") . "); must be 'fast' or 'normal' or 'high'\n"); } } if ((getGlobal("unitigger") ne "unitigger") && (getGlobal("unitigger") ne "bogart")) { addCommandLineError("ERROR: Invalid 'unitigger' specified (" . getGlobal("unitigger") . "); must be 'unitigger' or 'bogart'\n"); } if ((getGlobal("corConsensus") ne "utgcns") && (getGlobal("corConsensus") ne "falcon") && (getGlobal("corConsensus") ne "falconpipe")) { addCommandLineError("ERROR: Invalid 'corConsensus' specified (" . getGlobal("corConsensus") . "); must be 'utgcns' or 'falcon' or 'falconpipe'\n"); } if ((getGlobal("cnsConsensus") ne "quick") && (getGlobal("cnsConsensus") ne "pbdagcon") && (getGlobal("cnsConsensus") ne "utgcns")) { addCommandLineError("ERROR: Invalid 'cnsConsensus' specified (" . getGlobal("cnsConsensus") . "); must be 'quick', 'pbdagcon', or 'utgcns'\n"); } if ((!defined("lowCoverageAllowed") && defined("lowCoverageDepth")) || ( defined("lowCoverageAllowed") && !defined("lowCoverageDepth"))) { addCommandLineError("ERROR: Invalid 'lowCoverageAllowed' and 'lowCoverageDepth' specified; both must be set\n"); } #if ((getGlobal("cleanup") ne "none") && # (getGlobal("cleanup") ne "light") && # (getGlobal("cleanup") ne "heavy") && # (getGlobal("cleanup") ne "aggressive")) { # addCommandLineError("ERROR: Invalid cleaup specified (" . getGlobal("cleanup") . "); must be 'none', 'light', 'heavy' or 'aggressive'\n"); #} if ((getGlobal("corFilter") ne "quick") && (getGlobal("corFilter") ne "expensive") && (getGlobal("corFilter") ne "none")) { addCommandLineError("ERROR: Invalid 'corFilter' specified (" . getGlobal("corFilter") . "); must be 'none' or 'quick' or 'expensive'\n"); } if ((getGlobal("useGrid") ne "0") && (getGlobal("useGrid") ne "1") && (getGlobal("useGrid") ne "remote")) { addCommandLineError("ERROR: Invalid 'useGrid' specified (" . getGlobal("useGrid") . "); must be 'true', 'false' or 'remote'\n"); } if (defined(getGlobal("stopBefore"))) { my $ok = 0; my $st = getGlobal("stopBefore"); $st =~ tr/A-Z/a-z/; my $failureString = "ERROR: Invalid stopBefore specified (" . getGlobal("stopBefore") . "); must be one of:\n"; my @stopBefore = ("gatekeeper", "meryl", "trimReads", "splitReads", "red", "oea", "unitig", "consensusConfigure", "cns"); foreach my $sb (@stopBefore) { $failureString .= "ERROR: '$sb'\n"; $sb =~ tr/A-Z/a-z/; if ($st eq $sb) { $ok++; setGlobal('stopBefore', $st); } } addCommandLineError($failureString) if ($ok == 0); } if (defined(getGlobal("stopAfter"))) { my $ok = 0; my $st = getGlobal("stopAfter"); $st =~ tr/A-Z/a-z/; my $failureString = "ERROR: Invalid stopAfter specified (" . getGlobal("stopAfter") . "); must be one of:\n"; my @stopAfter = ("gatekeeper", "meryl", "mhapConfigure", "overlapStoreConfigure", "overlapStore", "unitig", "consensusConfigure", "consensusCheck", "consensusLoad", "consensusFilter"); foreach my $sa (@stopAfter) { $failureString .= "ERROR: '$sa'\n"; $sa =~ tr/A-Z/a-z/; if ($st eq $sa) { $ok++; setGlobal('stopAfter', $st); } } addCommandLineError($failureString) if ($ok == 0); } addCommandLineError("ERROR: Required parameter 'errorRate' is not set\n") if (! defined(getGlobal("errorRate"))); addCommandLineError("ERROR: Required parameter 'genomeSize' is not set\n") if (! defined(getGlobal("genomeSize"))); # # Java? Need JRE 1.8. # if ((getGlobal("corOverlapper") eq "mhap") || (getGlobal("obtOverlapper") eq "mhap") || (getGlobal("utgOverlapper") eq "mhap")) { my $java = getGlobal("java"); my $versionStr = "unknown"; my $version = 0; # Argh, we can't use runCommand() here, because we're included in Execution.pm. Try to check it with -x. # Nope. Fails if $java == "java". #if (! -x $java) { # addCommandLineError("ERROR: java executable '$java' not found or not executable\n"); #} open(F, "$java -showversion 2>&1 |"); while () { # First word is either "java" or "openjdk" or ... if (m/^.*\s+version\s+\"(\d+.\d+)(.*)\".*$/) { $versionStr = "$1$2"; $version = $1; } } close(F); if ($version < 1.8) { addCommandLineError("ERROR: mhap overlapper requires java version at least 1.8.0; you have $versionStr (from '$java').\n"); addCommandLineError("ERROR: '$java -showversion' reports:\n"); open(F, "$java -showversion 2>&1 |"); while () { chomp; addCommandLineError("ERROR: '$_'\n"); } close(F); } else { print STDERR "-- Detected Java(TM) Runtime Environment '$versionStr' (from '$java').\n"; } } # # Minimap, no valid identities, set legacy # if (getGlobal("corOverlapper") eq "minimap") { setGlobalIfUndef("corLegacyFilter", 1); } # # Falcon? Need to find it. # if ((getGlobal("corConsensus") eq "falcon") || (getGlobal("corConsensus") eq "falconpipe")) { my $falcon = getGlobal("falconSense"); addCommandLineError("ERROR: Didn't find falcon program with option falconSense='$falcon'") if ((defined($falcon)) && (! -e $falcon)); } # # Set default error rates based on the per-read error rate. # setGlobalIfUndef("corOvlErrorRate", 3.0 * getGlobal("errorRate")); setGlobalIfUndef("obtOvlErrorRate", 3.0 * getGlobal("errorRate")); setGlobalIfUndef("utgOvlErrorRate", 3.0 * getGlobal("errorRate")); setGlobalIfUndef("ovlErrorRate", 2.5 * getGlobal("errorRate")); setGlobalIfUndef("corsErrorRate", 10.0 * getGlobal("errorRate")); setGlobalIfUndef("cnsErrorRate", 2.5 * getGlobal("errorRate")); } 1; canu-1.3/src/pipelines/canu/ErrorEstimate.pm000066400000000000000000000236221272335422300211430ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Sergey Koren beginning on 2016-MAY-16 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::ErrorEstimate; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(estimateKmerError estimateRawError estimateCorrectedError uniqueKmerThreshold); use strict; use POSIX qw(floor); use File::Path qw(make_path remove_tree); use canu::Defaults; use canu::Execution; use canu::Gatekeeper; use canu::HTML; sub fac($) { my $x = shift @_; return 1 if($x == 0); return 1 if($x == 1); return $x * fac($x - 1); } sub poisson_pdf ($$) { my $lambda = shift @_; my $k = shift @_; return ( ( ($lambda ** $k) * exp(-$lambda) ) / fac($k) ); } sub uniqueKmerThreshold($$$$) { my $wrk = shift @_; my $asm = shift @_; my $merSize = shift @_; my $loss = shift @_; my $bin = getBinDirectory(); my $errorRate = estimateRawError($wrk, $asm, "cor", $merSize); my $readLength = getNumberOfBasesInStore($wrk, $asm) / getNumberOfReadsInStore ($wrk, $asm); my $effective_coverage = getExpectedCoverage($wrk, $asm) * ( ($readLength - $merSize + 1)/$readLength ) * (1 - $errorRate) ** $merSize; my $threshold = 0; my $kMer_loss = poisson_pdf($effective_coverage, 0); return 1 if($kMer_loss > $loss); my $keepTrying = 1; while($keepTrying) { $keepTrying = 0; my $p_true_kMers_threshold_p1 = poisson_pdf($effective_coverage, $threshold+1); if(($kMer_loss + $p_true_kMers_threshold_p1) <= $loss) { $threshold++; $kMer_loss += $p_true_kMers_threshold_p1; $keepTrying = 1; } } return ($threshold == 0 ? 1 : $threshold); } sub computeSampleSize($$$$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $percent = shift @_; my $coverage = shift @_; my $sampleSize = 0; my $minSampleSize = 100; my $maxSampleSize = getGlobal("${tag}MhapBlockSize") * 4; if (defined($percent)) { $sampleSize = int($percent * getNumberOfReadsInStore ($wrk, $asm))+1; $sampleSize++ if ($sampleSize % 2 != 0); } elsif (defined($coverage)) { $sampleSize = int(($coverage * getGlobal("genomeSize")) / (getNumberOfBasesInStore($wrk, $asm) / getNumberOfReadsInStore ($wrk, $asm))) + 1; } $sampleSize = $maxSampleSize if (defined($percent) && $sampleSize > $maxSampleSize); return $sampleSize < $minSampleSize ? $minSampleSize : $sampleSize; } sub runMHAP($$$$$$$$$$$$) { my ($wrk, $tag, $numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer, $sampleSize, $hash, $query, $out, $err) = @_; my $filterThreshold = getGlobal("${tag}MhapFilterThreshold"); my $merSize = getGlobal("${tag}MhapMerSize"); my $javaPath = getGlobal("java"); my $bin = getBinDirectory(); print STDERR "--\n"; print STDERR "-- PARAMETERS: hashes=$numHashes, minMatches=$minNumMatches, threshold=$threshold\n"; print STDERR "--\n"; my $cmd = "$javaPath -d64 -server -Xmx4g -jar $bin/mhap-" . getGlobal("${tag}MhapVersion") . ".jar "; $cmd .= " --no-self --repeat-weight 0.9 -k $merSize --num-hashes $numHashes --num-min-matches $minNumMatches --ordered-sketch-size $ordSketch --ordered-kmer-size $ordSketchMer --threshold $threshold --filter-threshold $filterThreshold --num-threads " . getGlobal("${tag}mhapThreads"); $cmd .= " -s $hash -q $query 2> /dev/null | awk '{if (\$1 != \$2+$sampleSize) { print \$0}}' | $bin/errorEstimate -d 2 -m 0.95 -S - > $out 2> $err"; runCommand($wrk, $cmd); } sub estimateRawError($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $merSize = shift @_; my $bin = getBinDirectory(); my $numReads = getNumberOfReadsInStore ($wrk, $asm); goto allDone if (skipStage($WRK, $asm, "errorEstimate") == 1); goto allDone if (-e "$wrk/asm.gkpStore/raw.estimate.out"); goto allDone if (getGlobal("errorrate") > 0); my ($numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer); $numHashes = 10000; $minNumMatches = 3; $threshold = 0.65; $ordSketch = 10000; $ordSketchMer = getGlobal("${tag}MhapOrderedMerSize"); # subsample raw reads my $sampleSize = computeSampleSize($wrk, $asm, $tag, 0.01, undef); $sampleSize /= 2; my $cmd = "$bin/gatekeeperDumpFASTQ -G $wrk/$asm.gkpStore -nolibname -fasta -r 1-$sampleSize -o - > $wrk/$asm.gkpStore/subset.fasta 2> /dev/null"; runCommandSilently($wrk, $cmd, 1); my $min = $numReads - $sampleSize + 1; my $cmd = "$bin/gatekeeperDumpFASTQ -G $wrk/$asm.gkpStore -nolibname -fasta -r $min-$numReads -o - >> $wrk/$asm.gkpStore/subset.fasta 2> /dev/null"; runCommandSilently($wrk, $cmd, 1); my $querySize = computeSampleSize($wrk, $asm, $tag, undef, 2); my $cmd = "$bin/gatekeeperDumpFASTQ -G $wrk/$asm.gkpStore -nolibname -fasta -r 1-$querySize -o - > $wrk/$asm.gkpStore/reads.fasta 2> /dev/null"; runCommandSilently($wrk, $cmd, 1); print STDERR "--\n"; print STDERR "-- ESTIMATOR (mhap) (raw) (hash sample size=". ($sampleSize*2) . ") (query sample size=$querySize)\n"; runMHAP($wrk, $tag, $numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer, $sampleSize*2, "$wrk/$asm.gkpStore/subset.fasta", "$wrk/$asm.gkpStore/reads.fasta", "$wrk/$asm.gkpStore/raw.estimate.out", "$wrk/$asm.gkpStore/raw.estimate.err"); unlink("$wrk/$asm.gkpStore/subset.fasta"); unlink("$wrk/$asm.gkpStore/reads.fasta"); allDone: return 0.15 if (! -e "$wrk/$asm.gkpStore/raw.estimate.out"); my $errorRate = 0; open(L, "< $wrk/$asm.gkpStore/raw.estimate.out") or caExit("can't open '$wrk/$asm.gkpStore/raw.estimate.out' for reading: $!", undef); while () { $errorRate = sprintf "%.3f", ($_ / 2); $errorRate = 0.15 if ($errorRate <= 0.005); } close(L); return $errorRate; } # Map subset of reads to long reads with mhap. # Compute resulting distribution and estimate error rate sub estimateCorrectedError ($$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $bin = getBinDirectory(); $wrk = "$wrk/correction"; my $path = "$wrk/3-estimator"; # only run if we aren't done and were asked to goto allDone if (skipStage($WRK, $asm, "errorEstimate") == 1); goto allDone if (-e "$path/$asm.estimate.out"); goto allDone if (getGlobal("errorrate") > 0); # Mhap parameters - filterThreshold needs to be a string, else it is printed as 5e-06. # my ($numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer); $numHashes = 256; $minNumMatches = 4; $threshold = 0.85; $ordSketch = 1000; $ordSketchMer = getGlobal("${tag}MhapOrderedMerSize") + 2; make_path("$path"); # subsample corrected reads, this assumes the fasta records are on a single line. We take some reads from the top and bottom of file to avoid sampling one library my $sampleSize = computeSampleSize($wrk, $asm, $tag, 0.01, undef); my $cmd = "gunzip -c $WRK/asm.correctedReads.fasta.gz |head -n $sampleSize > $path/subset.fasta"; runCommandSilently($path, $cmd, 1); my $cmd = "gunzip -c $WRK/asm.correctedReads.fasta.gz |tail -n $sampleSize >> $path/subset.fasta"; runCommandSilently($path, $cmd, 1); my $querySize = computeSampleSize($wrk, $asm, $tag, undef, 2); my $cmd = "gunzip -c $WRK/asm.correctedReads.fasta.gz |head -n $querySize > $path/reads.fasta"; runCommandSilently($path, $cmd, 1); my $cmd = "gunzip -c $WRK/asm.correctedReads.fasta.gz |tail -n $querySize >> $path/reads.fasta"; runCommandSilently($path, $cmd, 1); # now compute the overlaps print STDERR "--\n"; print STDERR "-- ESTIMATOR (mhap) (corrected) (hash sample size=$sampleSize) (query sample size=$querySize)\n"; runMHAP($wrk, $tag, $numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer, $sampleSize, "$path/subset.fasta", "$path/reads.fasta", "$path/$asm.estimate.out", "$path/$asm.estimate.err"); unlink("$path/subset.fasta"); unlink("$path/reads.fasta"); allDone: return if (! -e "$path/$asm.estimate.out"); my $errorRate = 0; open(L, "< $path/$asm.estimate.out") or caExit("can't open '$path/$asm.estimate.out' for reading: $!", undef); while () { $errorRate = sprintf "%.3f", ($_ / 2); } close(L); print STDERR "-- \n"; if ($errorRate > 0.13) { print STDERR "-- Estimated error rate: " . ($errorRate*100) . "% > " . (0.13 * 100) . "% limit, capping it.\n"; $errorRate = 0.13; } elsif ($errorRate < 0.005) { print STDERR "-- Estimated error rate: " . ($errorRate*100) . "%, increasing to " . (0.005 * 100). "%.\n"; $errorRate = 0.005; } else { print STDERR "-- Estimated error rate: " . ($errorRate * 100) . "%.\n"; } setErrorRate($errorRate); showErrorRates("-- "); print STDERR "-- \n"; } canu-1.3/src/pipelines/canu/Execution.pm000066400000000000000000001263421272335422300203240ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # kmer/ESTmapper/scheduler.pm # kmer/scripts/libBri.pm # kmer/scripts/scheduler.pm # src/pipelines/ca3g/Execution.pm # # Modifications by: # # Brian P. Walenz from 2003-JAN-03 to 2003-NOV-11 # are Copyright 2003 Applera Corporation, and # are subject to the GNU General Public License version 2 # # Brian P. Walenz on 2004-MAR-22 # are Copyright 2004 Brian P. Walenz, and # are subject to the GNU General Public License version 2 # # Brian P. Walenz from 2006-APR-07 to 2011-DEC-28 # are Copyright 2006,2008-2009,2011 J. Craig Venter Institute, and # are subject to the GNU General Public License version 2 # # Brian P. Walenz from 2015-FEB-27 to 2015-SEP-11 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-NOV-03 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-25 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Execution; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(stopBefore stopAfter skipStage emitStage touch getInstallDirectory getBinDirectory getBinDirectoryShellCode submitScript submitOrRunParallelJob runCommand runCommandSilently findCommand findExecutable); use strict; use Config; # for @signame use Cwd qw(getcwd); use POSIX ":sys_wait_h"; # For waitpid(..., &WNOHANG) use List::Util qw(min max); use File::Path qw(make_path remove_tree); use File::Spec; use canu::Defaults; # # Functions for running multiple processes at the same time. This is private to the module. # my $numberOfProcesses = 0; # Number of jobs concurrently running my $numberOfProcessesToWait = 0; # Number of jobs we can leave running at exit my @processQueue = (); my @processesRunning = (); my $printProcessCommand = 1; # Show commands as they run sub schedulerSetNumberOfProcesses { $numberOfProcesses = shift @_; } sub schedulerSubmit ($) { my $cmd = shift @_; chomp $cmd; push @processQueue, $cmd; } sub schedulerForkProcess ($) { my $process = shift @_; my $pid; # From Programming Perl, page 167 FORK: { if ($pid = fork) { # Parent # return($pid); } elsif (defined $pid) { # Child # exec($process); } elsif ($! =~ /No more processes/) { # EAGIN, supposedly a recoverable fork error sleep 1; redo FORK; } else { die "Can't fork: $!\n"; } } } sub schedulerReapProcess ($) { my $pid = shift @_; if (waitpid($pid, &WNOHANG) > 0) { return(1); } else { return(0); } } sub schedulerRun () { my @running; # Reap any processes that have finished foreach my $i (@processesRunning) { push @running, $i if (schedulerReapProcess($i) == 0); } @processesRunning = @running; # Run processes in any available slots while ((scalar(@processesRunning) < $numberOfProcesses) && (scalar(@processQueue) > 0)) { my $process = shift @processQueue; print STDERR " $process\n"; push @processesRunning, schedulerForkProcess($process); } } sub schedulerFinish ($) { my $dir = shift @_; my $child; my @newProcesses; my $remain; $remain = scalar(@processQueue); my $startsecs = time(); my $diskfree = (defined($dir)) ? (diskSpace($dir)) : (0); print STDERR "----------------------------------------\n"; print STDERR "-- Starting concurrent execution on ", scalar(localtime()), " with $diskfree GB free disk space ($remain processes; $numberOfProcesses concurrently)\n" if (defined($dir)); print STDERR "-- Starting concurrent execution on ", scalar(localtime()), " ($remain processes; $numberOfProcesses concurrently)\n" if (!defined($dir)); print STDERR "\n"; # Run all submitted jobs # while ($remain > 0) { schedulerRun(); $remain = scalar(@processQueue); if ($remain > 0) { $child = waitpid -1, 0; undef @newProcesses; foreach my $i (@processesRunning) { push @newProcesses, $i if ($child != $i); } undef @processesRunning; @processesRunning = @newProcesses; } } # Wait for them to finish, if requested # while (scalar(@processesRunning) > $numberOfProcessesToWait) { waitpid(shift @processesRunning, 0); } $diskfree = (defined($dir)) ? (diskSpace($dir)) : (0); my $warning = " !!! WARNING !!!" if ($diskfree < 10); my $elapsed = time() - $startsecs; $elapsed = "lickety-split" if ($elapsed eq "0"); $elapsed = "$elapsed second" if ($elapsed eq "1"); $elapsed = "$elapsed seconds" if ($elapsed > 1); print STDERR "\n"; print STDERR "-- Finished on ", scalar(localtime()), " ($elapsed) with $diskfree GB free disk space$warning\n"; print STDERR "----------------------------------------\n"; } # # File Management # sub touch ($@) { open(F, "> $_[0]") or caFailure("failed to touch file '$_[0]'", undef); print F "$_[1]\n" if (defined($_[1])); close(F); } # # State management # sub stopBefore ($$) { my $stopBefore = shift @_; my $cmd = shift @_; $stopBefore =~ tr/A-Z/a-z/; if ((defined($stopBefore)) && (defined(getGlobal("stopBefore"))) && (getGlobal("stopBefore") eq $stopBefore)) { print STDERR "\n"; print STDERR "Stop requested before '$stopBefore'.\n"; print STDERR "\n"; print STDERR "Command:\n $cmd\n" if (defined($cmd)); exit(0); } } sub stopAfter ($) { my $stopAfter = shift @_; $stopAfter =~ tr/A-Z/a-z/; if ((defined($stopAfter)) && (defined(getGlobal("stopAfter"))) && (getGlobal("stopAfter") eq $stopAfter)) { print STDERR "Stop requested after '$stopAfter'.\n"; exit(0); } } sub lookupStageLabel ($) { my $label = shift @_; my %ckp; my $index; # For correction $index = 100; $ckp{'cor-gatekeeper'} = $index++; $ckp{'cor-meryl'} = $index++; $ckp{'cor-mhapConfigure'} = $index++; $ckp{'cor-mhapPrecomputeCheck'} = $index++; # + attempt $ckp{'cor-mhapCheck'} = $index++; # + attempt $ckp{'cor-overlapConfigure'} = $index++; $ckp{'cor-overlapCheck'} = $index++; # + attempt $ckp{'cor-overlapStoreConfigure'} = $index++; $ckp{'cor-overlapStoreBucketizerCheck'} = $index++; $ckp{'cor-overlapStoreSorterCheck'} = $index++; $ckp{'cor-createOverlapStore'} = $index++; $ckp{'cor-buildCorrectionLayouts'} = $index++; $ckp{'cor-generateCorrectedReads'} = $index++; # + attempt $ckp{'cor-dumpCorrectedReads'} = $index++; # For trimming $index = 200; $ckp{'obt-gatekeeper'} = $index++; $ckp{'obt-meryl'} = $index++; $ckp{'obt-mhapConfigure'} = $index++; $ckp{'obt-mhapPrecomputeCheck'} = $index++; # + attempt $ckp{'obt-overlapConfigure'} = $index++; $ckp{'obt-overlapCheck'} = $index++; # + attempt $ckp{'obt-overlapStoreConfigure'} = $index++; $ckp{'obt-overlapStoreBucketizerCheck'} = $index++; $ckp{'obt-overlapStoreSorterCheck'} = $index++; $ckp{'obt-createOverlapStore'} = $index++; $ckp{'obt-trimReads'} = $index++; $ckp{'obt-splitReads'} = $index++; $ckp{'obt-dumpReads'} = $index++; # rename this # For assembly $index = 300; $ckp{'utg-gatekeeper'} = $index++; $ckp{'utg-meryl'} = $index++; $ckp{'utg-mhapConfigure'} = $index++; $ckp{'utg-mhapPrecomputeCheck'} = $index++; # + attempt $ckp{'utg-overlapConfigure'} = $index++; $ckp{'utg-overlapCheck'} = $index++; # + attempt $ckp{'utg-overlapStoreConfigure'} = $index++; $ckp{'utg-overlapStoreBucketizerCheck'} = $index++; $ckp{'utg-overlapStoreSorterCheck'} = $index++; $ckp{'utg-createOverlapStore'} = $index++; $ckp{'overlapFilterDetectConfigure'} = $index++; $ckp{'overlapFilterDetectCheck'} = $index++; # + attempt $ckp{'overlapFilterConfigure'} = $index++; $ckp{'overlapFilter'} = $index++; # + attempt $ckp{'readErrorDetectionConfigure'} = $index++; $ckp{'readErrorDetectionCheck'} = $index++; # + attempt $ckp{'overlapErrorAdjustmentConfigure'} = $index++; $ckp{'overlapErrorAdjustmentCheck'} = $index++; # + attempt $ckp{'updateOverlapStore'} = $index++; $ckp{'unitig'} = $index++; $ckp{'consensusConfigure'} = $index++; $ckp{'consensusCheck'} = $index++; # + attempt $ckp{'consensusLoad'} = $index++; $ckp{'consensusFilter'} = $index++; $ckp{'outputLayout'} = $index++; $ckp{'outputGraph'} = $index++; $ckp{'outputSequence'} = $index++; caFailure("invalid checkpoint label '$label'", undef) if (!defined($ckp{$label})); return($ckp{$label}); } # Returns true if we should skip this stage. No used, but left in for possible use in cleaning up things. Signals that we're # at the start of some stage, and we could clean up earlier stages. # sub skipStage ($$$@) { my $wrk = shift @_; my $asm = shift @_; my $stage = shift @_; my $attempt = shift @_; my $ckpstage = ""; my $ckpattempt = undef; # DISABLED. return(0); if (! -e "$wrk/$asm.stage") { # No checkpoint file exists, must compute! print STDERR "No $wrk/$asm.stage file, compute it all!\n"; return(0); } open(F, "< $wrk/$asm.stage") or caFailure("failed to open '$wrk/$asm.stage' for reading", undef); while () { if (m/canu\s+at\s+stage\s+(\S*)\s+\(#\d+\)\sattempt\s+(\d+)$/) { $ckpstage = $1; $ckpattempt = $2; } if (m/canu\s+at\s+stage\s+(\S*)\s+\(#\d+\)$/) { $ckpstage = $1; } } close(F); caFailure("didn't find stage in '$wrk/$asm.stage'", undef) if ($ckpstage eq ""); # Don't skip it. The stage to run is after the stage in the checkpoint. if (lookupStageLabel($ckpstage) < lookupStageLabel($stage)) { #print STDERR "PURGE at stage $stage (ignored attempt $attempt)\n"; purgeRecomputable($wrk, $asm, $stage); return(0); }; # Don't skip it. The stage to run is the stage in the checkpoint, but the attempt we're trying # is after. if ((lookupStageLabel($stage) == lookupStageLabel($ckpstage) && (defined($attempt)) && (defined($ckpattempt)) && ($ckpattempt < $attempt))) { #print STDERR "PURGE at stage $stage attempt $attempt\n"; purgeRecomputable($wrk, $asm, $stage); return(0); }; #print STDERR "skipStage()-- target $stage/" . lookupStageLabel($stage); #print STDERR " attempt $attempt" if (defined($attempt)); #print STDERR " -- checkpoint $ckpstage/" . lookupStageLabel($ckpstage); #print STDERR " attempt $ckpattempt" if (defined($ckpattempt)); #print STDERR " -- from '$wrk/$asm.stage'\n"; # Skip it. But first, purge any files we'll never need again. print STDERR "PURGE extraneous before stage $stage attempt $attempt\n"; purgeExtraneous($wrk, $asm, $stage); return(1); } # Same as skipStage(), left in for future use cleaning up. Signals that we're done with a stage. # sub emitStage ($$$@) { return; my $wrk = shift @_; my $asm = shift @_; my $stage = shift @_; my $attempt = shift @_; my $time = localtime(); my $label = lookupStageLabel($stage); my $label1 = $label - 1; my $attempt = (defined($attempt)) ? " attempt $attempt" : ""; my $ATTEMPT = (defined($attempt)) ? " ATTEMPT $attempt" : ""; open(F, ">> $wrk/$asm.stage") or caFailure("failed to open '$wrk/$asm.stage' for appending\n", undef); print F "$time -- canu at stage $stage (#$label)$attempt\n"; close(F); #print "----------------------------------------STAGE $stage (#$label)$ATTEMPT FINISHED.\n"; make_path("$wrk/$asm.stage.fileLists") if (! -d "$wrk/$asm.stage.fileLists"); # Find all files created since the last checkpoint, or accessed since the last. Linux claims either -anewer or -newera will work # # Format is: access-time -- modification-time -- status-change-time -- filename # Problem - label-1 doesn't always exist because we occasionally reset the label to 200, 300. Fixed by not doing that. while (($label1 > 100) && (! -e "$wrk/$asm.stage.fileLists/stage.$label1.created")) { $label1--; } if (-e "$wrk/$asm.stage.fileLists/stage.$label1.created") { runCommandSilently($wrk, "find . -type f -and -newer $wrk/$asm.stage.fileLists/stage.$label1.created -print > $wrk/$asm.stage.fileLists/stage.$label.created", 1); runCommandSilently($wrk, "find . -type f -and -anewer $wrk/$asm.stage.fileLists/stage.$label1.created -print > $wrk/$asm.stage.fileLists/stage.$label.accessed", 1); } else { runCommandSilently($wrk, "find . -type f -print > $wrk/$asm.stage.fileLists/stage.$label.created", 1); } if (! -e "$wrk/$asm.stage.fileLists/stage.$label1.created") { caExit("failed to generate list of files created since last checkpoint", undef); } } # #my %firstAccessed; #my %lastAccessed; # # #sub readAccessed ($$$$) { # my $wrk = shift @_; # my $asm = shift @_; # my $stage = shift @_; # my $file = shift @_; # # open(G, "< $file") or die "Failed to open '$file' for reading: $!\n"; # while () { # chomp; # # s!^\.\/!!; # s!$asm\.!PREFIX.!; # #s!\d\d\d\d\d\d\.out!TASKID.out!; # s!\d\d\d\d\d\d\.!TASKID.!; # # next if (m/PREFIX.stage.fileLists/); # next if (m/PREFIX.stage/); # next if (m/canu-logs/); # # $firstAccessed{$_} = $stage if (!exists($firstAccessed{$_})); # $lastAccessed{$_} = $stage; # } # close(G); #} # # ## Remove files we won't ever need again. #sub purgeExtraneous ($$$$) { # my $wrk = shift @_; # my $asm = shift @_; # my $stage = shift @_; # # open(F, "ls $wrk/$asm.stage.fileLists/ |"); # while () { # my $file = $_; chomp $file; # # if ($file =~ m/stage.(\d+).accessed/) { # readAccessed($wrk, $asm, $1, "$wrk/test.stage.fileLists/$file"); # } # } # close(F); # # foreach my $f (keys %lastAccessed) { # next if ($lastAccessed{$f} < $stage); # # #print STDERR "AT stage $stage REMOVE extraneous file $f last needed in stage $lastAccessed{$f}\n"; # } #} # # # ## Remove files we'll be computing again. This is based on knowing the current stage we're at, ## then examining the sge.fileLists directory for future stages and removing those files. ## ## If any are detected, we emit a new stage (to find files between the last stage and the last stop), ## then delete those files. ## #sub purgeRecomputable ($$$) { # my $wrk = shift @_; # my $asm = shift @_; # my $stage = shift @_; # # open(F, "ls $wrk/$asm.stage.fileLists/ |"); # while () { # my $file = $_; chomp $file; # # if ($file =~ m/stage.(\d+).accessed/) { # readAccessed($wrk, $asm, $1, "$wrk/test.stage.fileLists/$file"); # } # } # close(F); # # foreach my $f (keys %firstAccessed) { # next if ($stage < $firstAccessed{$f}); # # print STDERR "AT stage $stage REMOVE premature file $f first needed in stage $firstAccessed{$f}\n"; # } #} # # Decide what bin directory to use. # # When we are running on the grid, the path of this perl script is NOT always the correct # architecture. If the submission host is FreeBSD, but the grid is Linux, the BSD box will submit # FreeBSD/bin/canu to the grid. Unless it knows which grid host it will run on in advance, there # is no way to pick the correct one. The grid host then has to have enough smarts to choose the # correct binaries, and that is what we're doing here. # # To make it more trouble, shell scripts need to do all this by themselves. # sub getInstallDirectory () { my $installDir = $FindBin::RealBin; if ($installDir =~ m!^(.*)/\w+-\w+/bin$!) { $installDir = $1; } return($installDir); } # Used inside canu to find where binaries are located. It uses uname to find OS, architecture and # system name, then uses that to construct a path to binaries. If a "pathMap" is defined, this is # used to hardcode a path to a system name. # sub getBinDirectory () { my $installDir = getInstallDirectory(); my $syst = `uname -s`; chomp $syst; # OS implementation my $arch = `uname -m`; chomp $arch; # Hardware platform my $name = `uname -n`; chomp $name; # Name of the system $arch = "amd64" if ($arch eq "x86_64"); $arch = "ppc" if ($arch eq "Power Macintosh"); my $path = "$installDir/$syst-$arch/bin"; my $pathMap = getGlobal("pathMap"); if (defined($pathMap)) { open(F, "< $pathMap") or caFailure("failed to open pathMap '$pathMap'", undef); while () { my ($n, $b) = split '\s+', $_; $path = $b if ($name eq $n); } close(F); } if (! -d "$path") { $path = $installDir; } return($path); } # Emits a block of shell code to locate binaries during shell scripts. See comments on # getBinDirectory. # sub getBinDirectoryShellCode () { my $installDir = getInstallDirectory(); my $string; $string .= "syst=`uname -s`\n"; $string .= "arch=`uname -m`\n"; $string .= "name=`uname -n`\n"; $string .= "\n"; $string .= "if [ \"\$arch\" = \"x86_64\" ] ; then\n"; $string .= " arch=\"amd64\"\n"; $string .= "fi\n"; $string .= "if [ \"\$arch\" = \"Power Macintosh\" ] ; then\n"; $string .= " arch=\"ppc\"\n"; $string .= "fi\n"; $string .= "\n"; $string .= "bin=\"$installDir/\$syst-\$arch/bin\"\n"; $string .= "\n"; my $pathMap = getGlobal("pathMap"); if (defined($pathMap)) { open(PM, "< $pathMap") or caFailure("failed to open pathMap '$pathMap'", undef); while () { my ($n, $b) = split '\s+', $_; $string .= "if [ \"\$name\" = \"$n\" ] ; then\n"; $string .= " bin=\"$b\"\n"; $string .= "fi\n"; } close(PM); $string .= "\n"; } $string .= "if [ ! -d \"\$bin\" ] ; then\n"; $string .= " bin=\"$installDir\"\n"; $string .= "fi\n"; $string .= "\n"; return($string); } # Spend too much effort ensuring that the name is unique in the system. For 'canu' jobs, we don't # care. sub makeRandomSuffix ($) { my $length = shift @_; my @chars = +('0'..'9', 'a'..'k', 'm'..'z', 'A'..'H', 'J'..'N', 'P'..'Z'); # Remove 'l', 'I' and 'O' my $suffix; while ($length-- > 0) { $suffix .= @chars[int(rand(59))]; } return($suffix); } sub makeUniqueJobName ($$) { my $jobType = shift @_; my $asm = shift @_; # If a canu job, just return the standard name. No uniquification needed. if ($jobType eq "canu") { return("canu_" . $asm . ((defined(getGlobal("gridOptionsJobName"))) ? ("_" . getGlobal("gridOptionsJobName")) : (""))); } # For all other jobs, we need to ensure the name is unique. We do this by adding digits at the end. my $jobName = "${jobType}_" . $asm . ((defined(getGlobal("gridOptionsJobName"))) ? ("_" . getGlobal("gridOptionsJobName")) : ("")); my %jobs; # First, find the list of all jobs that exist. if (getGlobal("gridEngine") eq "SGE") { open(F, "qstat -xml |"); while () { $jobs{$1}++ if (m/^\s*(.*)<\/JB_name>$/); } close(F); } if (getGlobal("gridEngine") eq "PBS") { } if (getGlobal("gridEngine") eq "LSF") { } # If the jobName doesn't exist, we can use it. return($jobName) if (! exists($jobs{$jobName})); # Otherwise, find a unique random 2-letter suffix. my $jobIdx = makeRandomSuffix(2); while (exists($jobs{"${jobName}_$jobIdx"})) { $jobIdx = makeRandomSuffix(2); } # And return it! Simple! # this was breaking dependencies when multiple jobs were submitted like for a failed consensus run, turn off for now return("${jobName}"); #return("${jobName}_$jobIdx"); } # Submit ourself back to the grid. If the one argument is defined, make us hold on jobs with that # name. # # The previous version (CA) would use "gridPropagateHold" to reset holds on existing jobs so that # they would also hold on this job. # sub submitScript ($$$) { my $wrk = shift @_; my $asm = shift @_; my $jobToWaitOn = shift @_; return if (getGlobal("useGrid") ne "1"); # If not requested to run on the grid, return if (getGlobal("gridEngine") eq undef); # or can't run on the grid, don't run on the grid. # If no job to wait on, and we are already on the grid, do NOT resubmit ourself. # # When the user launches canu on the head node, a call to submitScript() is made to launch canu # under grid control. That results in a restart of canu, and another call to submitScript(), # but this time, the envorinment variable is set, we we can skip the resubmission, and continue # with canu execution. return if (($jobToWaitOn eq undef) && (exists($ENV{getGlobal("gridEngineJobID")}))); # Find the next available output file. make_path("$wrk/canu-scripts") if (! -d "$wrk/canu-scripts"); # Done in canu.pl, just being paranoid my $idx = "01"; while (-e "$wrk/canu-scripts/canu.$idx.out") { $idx++; } my $output = "$wrk/canu-scripts/canu.$idx.out"; my $script = "$wrk/canu-scripts/canu.$idx.sh"; my $iteration = getGlobal("canuIteration"); # Make a script for us to submit. open(F, "> $script") or caFailure("failed to open '$script' for writing", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n" if (getGlobal("gridEngine") eq "SGE"); print F "# Attempt to (re)configure SGE. For unknown reasons, jobs submitted\n" if (getGlobal("gridEngine") eq "SGE"); print F "# to SGE, and running under SGE, fail to read the shell init scripts,\n" if (getGlobal("gridEngine") eq "SGE"); print F "# and so they don't set up SGE (or ANY other paths, etc) properly.\n" if (getGlobal("gridEngine") eq "SGE"); print F "# For the record, interactive logins (qlogin) DO set the environment.\n" if (getGlobal("gridEngine") eq "SGE"); print F "\n"; print F "if [ \"x\$SGE_ROOT\" != \"x\" ]; then \n" if (getGlobal("gridEngine") eq "SGE"); print F " . \$SGE_ROOT/\$SGE_CELL/common/settings.sh\n" if (getGlobal("gridEngine") eq "SGE"); print F "fi\n" if (getGlobal("gridEngine") eq "SGE"); print F "\n"; print F "# On the off chance that there is a pathMap, and the host we\n"; print F "# eventually get scheduled on doesn't see other hosts, we decide\n"; print F "# at run time where the binary is.\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; print F "/usr/bin/env perl \\\n"; print F "\$bin/canu " . getCommandLineOptions() . " canuIteration=$iteration\n"; close(F); system("chmod +x $script"); # Construct a submission command line. my ($jobName, $memOption, $thrOption, $gridOpts); $jobName = makeUniqueJobName("canu", $asm); # The canu.pl script isn't expected to take resources. We'll default to 4gb and one thread. my $mem = 4; my $thr = 1; # However, the sequential overlap store is still built from within the canu process. if (getGlobal("ovsMethod") eq "sequential") { $mem = max($mem, getGlobal("ovsMemory")); } $memOption = buildMemoryOption($mem, 1); $thrOption = buildThreadOption($thr); $gridOpts = $memOption if (defined($memOption)); $gridOpts .= " " if (defined($gridOpts)); $gridOpts .= $thrOption if (defined($thrOption)); $gridOpts .= " " if (defined($gridOpts)); $gridOpts .= getGlobal("gridOptions") if (defined(getGlobal("gridOptions"))); $gridOpts .= " " if (defined($gridOpts)); $gridOpts .= getGlobal("gridOptionsExecutive") if (defined(getGlobal("gridOptionsExecutive"))); # If the jobToWaitOn is defined, make the script wait for that to complete. LSF might need to # query jobs in the queue and figure out the job ID (or IDs) for the jobToWaitOn. Reading LSF # docs online (for bsub.1) claim that we can still use jobToWaitOn. if (defined($jobToWaitOn)) { my $hold = getGlobal("gridEngineHoldOption"); # most grid engines don't understand job names to hold on, only IDs if (uc(getGlobal("gridEngine")) eq "PBS" || uc(getGlobal("gridEngine")) eq "SLURM"){ my $tcmd = getGlobal("gridEngineNameToJobIDCommand"); $tcmd =~ s/WAIT_TAG/$jobToWaitOn/g; my $propJobCount = `$tcmd |wc -l`; chomp $propJobCount; if ($propJobCount == 0) { $tcmd = getGlobal("gridEngineNameToJobIDCommandNoArray"); $tcmd =~ s/WAIT_TAG/$jobToWaitOn/g; $hold = getGlobal("gridEngineHoldOptionNoArray"); $propJobCount = `$tcmd |wc -l`; } if ($propJobCount != 1) { print STDERR "Warning: multiple IDs for job $jobToWaitOn got $propJobCount and should have been 1.\n"; } my $jobID = undef; open(F, "$tcmd |awk '{print \$1}' |"); while () { chomp $_; if (defined($jobID)) { $jobID = "$jobID:$_"; } else { $jobID = $_; } } close(F); $hold =~ s/WAIT_TAG/$jobID/g; } else { $hold =~ s/WAIT_TAG/$jobToWaitOn/; } $gridOpts .= " " . $hold; } my $submitCommand = getGlobal("gridEngineSubmitCommand"); my $nameOption = getGlobal("gridEngineNameOption"); my $outputOption = getGlobal("gridEngineOutputOption"); my $qcmd = "$submitCommand $gridOpts $nameOption \"$jobName\" $outputOption $output $script"; runCommand($wrk, $qcmd) and caFailure("Failed to submit script", undef); exit(0); } # Expects # job name # number of jobs # global pattern for option # sub buildGridArray ($$$$) { my $r = $_[3]; $r =~ s/ARRAY_NAME/$_[0]/g; # Replace ARRAY_NAME with 'job name' $r =~ s/ARRAY_JOBS/$_[1]-$_[2]/g; # Replace ARRAY_JOBS with 'bgn-end' return($r); } sub buildOutputName ($$$) { my $path = shift @_; my $script = shift @_; my $tid = shift @_; my $outName = "$path/$script.$tid.out"; if ((-e "$path/logs") && ($script =~ m/scripts\/(.*)/)) { $outName = "$path/logs/$1.$tid.out"; } return($outName); } sub buildMemoryOption ($$) { my $m = shift @_; my $t = shift @_; my $r; my $u = "g"; if (uc(getGlobal("gridEngine")) eq "SGE") { $m /= $t; } if ((uc(getGlobal("gridEngine")) eq "SLURM") && (getGlobal("gridEngineMemoryOption") =~ m/mem-per-cpu/i)) { $m /= $t; } if (int($m) != $m) { $m = int($m * 1024); $u = "m"; } if (uc(getGlobal("gridEngine")) eq "LSF") { $m = $m / 1024 if (getGlobal("gridEngineMemoryUnits") eq "t"); $m = $m * 1 if (getGlobal("gridEngineMemoryUnits") eq "g"); $m = $m * 1024 if (getGlobal("gridEngineMemoryUnits") eq "m"); $m = $m * 1024 * 1024 if (getGlobal("gridEngineMemoryUnits") eq "k"); $u = ""; } $r = getGlobal("gridEngineMemoryOption"); $r =~ s/MEMORY/${m}${u}/g; return($r); } sub buildThreadOption ($) { my $t = shift @_; my $r; $r = getGlobal("gridEngineThreadsOption"); $r =~ s/THREADS/$t/g; return($r); } sub buildGridJob ($$$$$$$$) { my $asm = shift @_; my $jobType = shift @_; my $path = shift @_; my $script = shift @_; my $mem = shift @_; my $thr = shift @_; my $bgnJob = shift @_; my $endJob = shift @_; # Unpack the job range if needed. if ($bgnJob =~ m/^(\d+)-(\d+)$/) { $bgnJob = $1; $endJob = $2; } if (!defined($endJob)) { $endJob = $bgnJob; } # Figure out the command and options needed to run the job. my $submitCommand = getGlobal("gridEngineSubmitCommand"); my $nameOption = getGlobal("gridEngineNameOption"); my $jobNameT = makeUniqueJobName($jobType, $asm); my $jobName = buildGridArray($jobNameT, $bgnJob, $endJob, getGlobal("gridEngineArrayName")); my $arrayOpt = buildGridArray($jobNameT, $bgnJob, $endJob, getGlobal("gridEngineArrayOption")); my $outputOption = getGlobal("gridEngineOutputOption"); my $outName = buildOutputName($path, $script, getGlobal("gridEngineArraySubmitID")); my $memOption = buildMemoryOption($mem, $thr); my $thrOption = buildThreadOption($thr); my $gridOpts; $gridOpts = getGlobal("gridOptions") if (defined(getGlobal("gridOptions"))); $gridOpts .= " " if (defined($gridOpts)); $gridOpts .= getGlobal("gridOptions$jobType") if (defined(getGlobal("gridOptions$jobType"))); $gridOpts .= " " if (defined($gridOpts)); $gridOpts .= $memOption if (defined($memOption)); $gridOpts .= " " if (defined($gridOpts)); $gridOpts .= $thrOption if (defined($thrOption)); # Build the command line. my $cmd; $cmd = " $submitCommand \\\n"; $cmd .= " $gridOpts \\\n" if (defined($gridOpts)); $cmd .= " $nameOption \"$jobName\" \\\n"; $cmd .= " $arrayOpt \\\n"; $cmd .= " $outputOption $outName \\\n"; $cmd .= " $path/$script.sh\n"; # Save it, just because. open(F, "> $path/$script.jobSubmit.sh") or die; print F $cmd; close(F); # Return the command and the job name it will be submitted with. return($cmd, $jobName); } # Convert @jobs to a list of ranges, a-b, c, d-e, etc. These will be directly submitted to the # grid, or run one-by-one locally. # # If we're SGE, we can combine everything to one job range: a-b,c,d,e-f. Except that # buildGridJob() doesn't know how to handle that. sub convertToJobRange (@) { my @jobs; # Expand the ranges into a simple list of job ids. foreach my $j (@_) { if ($j =~ m/^(\d+)-(\d+)$/) { for (my $a=$1; $a<=$2; $a++) { push @jobs, $a; } } elsif ($j =~ m/^(\d+)$/) { push @jobs, $1; } else { caFailure("invalid job format in '$j'", undef); } } # Sort. my @jobsA = sort { $a <=> $b } @jobs; undef @jobs; # Merge adjacent ids into a range. my $st = $jobsA[0]; my $ed = $jobsA[0]; shift @jobsA; foreach my $j (@jobsA) { if ($ed + 1 == $j) { $ed = $j; } else { push @jobs, ($st == $ed) ? "$st" : "$st-$ed"; $st = $j; $ed = $j; } } push @jobs, ($st == $ed) ? "$st" : "$st-$ed"; # If any of the ranges are larger than allowed, split into multiple pieces. my $l = getGlobal("gridEngineArrayMaxJobs") - 1; if ($l > 0) { @jobsA = @jobs; undef @jobs; foreach my $j (@jobsA) { if ($j =~ m/^(\d+)-(\d+)$/) { my $b = $1; my $e = $2; while ($b <= $e) { my $B = ($b + $l < $e) ? ($b + $l) : $e; push @jobs, "$b-$B"; $b += $l + 1; } } else { push @jobs, $j } } undef @jobsA; } return(@jobs); } # Expects # job type ("ovl", etc) # output directory # script name with no directory or .sh # number of jobs in the task # # If under grid control, submit grid jobs. Otherwise, run in parallel locally. # sub submitOrRunParallelJob ($$$$$@) { my $wrk = shift @_; # Root of the assembly (NOT wrk/correction or wrk/trimming) my $asm = shift @_; # Name of the assembly my $jobType = shift @_; # E.g., ovl, cns, ... - populates 'gridOptionsXXX # - also becomes the grid job name prefix, so three letters suggested my $path = shift @_; # Location of script to run my $script = shift @_; # Runs $path/$script.sh > $path/$script.######.out my $mem = getGlobal("${jobType}Memory"); my $thr = getGlobal("${jobType}Threads"); my @jobs = convertToJobRange(@_); # The script MUST be executable. system("chmod +x \"$path/$script.sh\""); # Report what we're doing. #my $t = localtime(); #print STDERR "----------------------------------------GRIDSTART $t\n"; #print STDERR "$path/$script.sh with $mem gigabytes memory and $thr threads.\n"; # Check stopping rules. stopBefore($jobType, "$path/$script.sh"); # Break infinite loops. If the grid jobs keep failing, give up after a few attempts. # # submitScript() passes canuIteration on to the next call. # canuIteration is reset to zero if the Check() for any parallel step succeeds. # # Assuming grid jobs die on each attempt: # 0) canu run from the command line submits iteration 1; canuIteration is NOT incremented # because no parallel jobs have been submitted. # 1) Iteration 1 - canu.pl submits jobs, increments the interation count, and submits itself as iteration 2 # 2) Iteration 2 - canu.pl submits jobs, increments the interation count, and submits itself as iteration 3 # 3) Iteration 3 - canu.pl fails with the error below # # If the jobs succeed in Iteration 2, the canu in iteration 3 will pass the Check(), never call # this function, and continue the pipeline. caExit("canu iteration count too high, stopping pipeline (most likely a problem in the grid-based computes)", undef) if (getGlobal("canuIteration") > getGlobal("canuIterationMax")); setGlobal("canuIteration", getGlobal("canuIteration") + 1); # If 'gridEngineJobID' environment variable exists (SGE: JOB_ID; LSF: LSB_JOBID) then we are # currently running under grid crontrol. If so, run the grid command to submit more jobs, then # submit ourself back to the grid. If not, tell the user to run the grid command by hand. # Jobs under grid control, and we submit them if (defined(getGlobal("gridEngine")) && (getGlobal("useGrid") eq "1") && (getGlobal("useGrid$jobType") eq "1") && (exists($ENV{getGlobal("gridEngineJobID")}))) { my $cmd; my $jobName; foreach my $j (@jobs) { ($cmd, $jobName) = buildGridJob($asm, $jobType, $path, $script, $mem, $thr, $j, undef); runCommand($path, $cmd) and caFailure("Failed to submit batch jobs", undef); } submitScript($wrk, $asm, $jobName); # submitScript() should never return. If it does, then a parallel step was attempted too many time. caExit("Too many attempts to run a parallel stage on the grid. Stop.", undef); } # Jobs under grid control, but the user must submit them if (defined(getGlobal("gridEngine")) && (getGlobal("useGrid") ne "0") && (getGlobal("useGrid$jobType") eq "1") && (! exists($ENV{getGlobal("gridEngineJobID")}))) { print STDERR "\n"; print STDERR "Please submit the following jobs to the grid for execution using $mem gigabytes memory and $thr threads:\n"; print STDERR "\n"; foreach my $j (@jobs) { my ($cmd, $jobName) = buildGridJob($asm, $jobType, $path, $script, $mem, $thr, $j, undef); print $cmd; } print STDERR "\n"; print STDERR "When all jobs complete, restart canu as before.\n"; exit(0); } # Standard jobs, run locally. my $cwd = getcwd(); # Remember where we are. chdir($path); # So we can root the jobs in the correct location. foreach my $j (@jobs) { my $st; my $ed; if ($j =~ m/^(\d+)-(\d+)$/) { $st = $1; $ed = $2; } else { $st = $ed = $j; } for (my $i=$st; $i<=$ed; $i++) { my $outName = buildOutputName($path, $script, substr("000000" . $i, -6)); schedulerSubmit("$path/$script.sh $i > $outName 2>&1"); } } # compute limit based on # of cpus my $nCParallel = getGlobal("${jobType}Concurrency"); $nCParallel = int(getGlobal("maxThreads") / $thr) if ((!defined($nCParallel)) || ($nCParallel == 0)); $nCParallel = 1 if ((!defined($nCParallel)) || ($nCParallel == 0)); # compute limit based on physical memory my $nMParallel = getGlobal("${jobType}Concurrency"); $nMParallel = int(getGlobal("maxMemory") / getGlobal("${jobType}Memory")) if ((!defined($nMParallel)) || ($nMParallel == 0)); $nMParallel = 1 if ((!defined($nMParallel)) || ($nMParallel == 0)); # run min of our limits my $nParallel = $nCParallel < $nMParallel ? $nCParallel : $nMParallel; schedulerSetNumberOfProcesses($nParallel); schedulerFinish($path); chdir($cwd); } # Pretty-ify the command. If there are no newlines already in it, break # before every switch and before file redirects. sub prettifyCommand ($) { my $dis = shift @_; if (($dis =~ tr/\n/\n/) == 0) { $dis =~ s/\s-/ \\\n -/g; # Replace ' -' with '\n -' (newline, two spaces, then the dash) $dis =~ s/\s>\s/ \\\n> /; # Replace ' > ' with '\n> ' $dis =~ s/\s2>\s/ \\\n2> /; # Replace ' 2> ' with '\n2> ' } $dis = " " . $dis; # Indent the command by four spaces. $dis =~ s/\n/\n /g; return($dis); } sub reportRunError ($) { my $rc = shift @_; # Bunch of busy work to get the names of signals. Is it really worth it?! my @signame; if (defined($Config{sig_name})) { my $i = 0; foreach my $n (split('\s+', $Config{sig_name})) { $signame[$i] = $n; $i++; } } else { for (my $i=0; $i<127; $i++) { $signame[$i] = "signal $i"; } } # The rest is rather straightforward at least. print STDERR "ERROR:\n"; if ($rc == -1) { print STDERR "ERROR: Failed to run the command. (rc=$rc)\n"; } elsif ($rc & 127) { print STDERR "ERROR: Failed with signal $signame[$rc & 127]. (rc=$rc)\n"; } else { print STDERR "ERROR: Failed with exit code ", $rc >> 8 , ". (rc=$rc)\n"; } print STDERR "ERROR:\n"; } # Utility to run a command and check the exit status, report time used. # sub runCommand ($$) { my $dir = shift @_; my $cmd = shift @_; my $dis = prettifyCommand($cmd); return if ($cmd eq ""); # Check if the directory exists. if (! -d $dir) { caFailure("Directory '$dir' doesn't exist, can't run command", ""); } # If only showing the next command, show it and stop. if (getGlobal("showNext")) { print STDERR "--NEXT-COMMAND\n"; print STDERR "$dis\n"; exit(0); } # Log that we're starting, and show the pretty-ified command. my $startsecs = time(); my $diskfree = (defined($dir)) ? (diskSpace($dir)) : (0); print STDERR "----------------------------------------\n"; print STDERR "-- Starting command on ", scalar(localtime()), " with $diskfree GB free disk space\n" if (defined($dir)); print STDERR "-- Starting command on ", scalar(localtime()), "\n" if (!defined($dir)); print STDERR "\n"; print STDERR "$dis\n"; my $rc = 0xffff & system("cd $dir && $cmd"); $diskfree = (defined($dir)) ? (diskSpace($dir)) : (0); my $warning = " !!! WARNING !!!" if ($diskfree < 10); my $elapsed = time() - $startsecs; $elapsed = "lickety-split" if ($elapsed eq "0"); $elapsed = "$elapsed second" if ($elapsed eq "1"); $elapsed = "$elapsed seconds" if ($elapsed > 1); print STDERR "\n"; print STDERR "-- Finished on ", scalar(localtime()), " ($elapsed) with $diskfree GB free disk space$warning\n"; print STDERR "----------------------------------------\n"; # Pretty much copied from Programming Perl page 230 return(0) if ($rc == 0); reportRunError($rc); return(1); } sub runCommandSilently ($$$) { my $dir = shift @_; my $cmd = shift @_; my $dis = prettifyCommand($cmd); my $critical = shift @_; return(0) if ($cmd eq ""); if (! -d $dir) { caFailure("Directory '$dir' doesn't exist, can't run command", ""); } my $rc = 0xffff & system("cd $dir && $cmd"); return(0) if ($rc == 0); # No errors, return no error. return(1) if ($critical == 0); # If not critical, return that it failed, otherwise, report error and fail. print STDERR "$dis\n"; reportRunError($rc); return(1); } sub findCommand ($) { my $cmd = shift @_; my @path = File::Spec->path; for my $path (@path) { if (-x "$path/$cmd") { return("$path/$cmd"); } } return(undef); } sub findExecutable ($) { my $exec = shift @_; my $path = `which \"$exec\" 2> /dev/null`; $path =~ s/^\s+//; $path =~ s/\s+$//; return(undef) if ($path eq ""); return($path); } 1; canu-1.3/src/pipelines/canu/Execution.txt000066400000000000000000000052041272335422300205200ustar00rootroot00000000000000 Running Commands ---------------- Simple single commands can be run, and logged to the chatter output, with something like if (output-doesn't-exist) { if (runCommand(directory, command)) { caExit(command-failed-message, command.err) } do-steps-to-make-output-exist } If no chatter output is desired, runCommandSilently() can be used. Ideally in the same recipe as above, but usually it isn't guarded at all. This function will terminate ungracefully if the command fails. For jobs to be run on the grid, either in parallel or a single job, the function submitOrRunParallelJob() is used. This takes a job type (as in gridOptions{jobType}), a path and a script, and a list of numeric job IDs to run. The list can be formed of simple integers or ranges, or both (e.g., 1,2,3-9,10,12-99). Its use is straightforward, but the wrapper to make it work both for grid-based execution and local execution is non-trivial. useGrid=remote fails maxMemory / maxThreads minMemory / minThreads - means what? maxGridCores - sge "-tc N", slurm/pbs "-a 1-1000%50" A *Configure() function needs to prepare the job. Its product is a shell script to run the job. A *Check() function parses the shell script (usually) to find out which jobs to run, then submitOrRunParallelJob() to execute them. Each Check() function is called up to three times (for a MaxIteration=2) - the first two to actually try to compute and the last to fail. For executions not using the grid, the check function will run the jobs and fall through to the finishStage: clause, reporting the job finished and maybe generating some stats. If the job fails, it is retried, using the same flow. For executions using the grid, the check function breaks execution and checking into two grid jobs. A parallel job runs the compute, and a sequential job holds on the parallel job. The sequential job remembers canuIteration (by having it passed on the command line as a parameter). If the parallel jobs succeeded, it falls through to finishStage: as above. If they had failed, they are tried again. The flow of each *Check() function is: goto allDone if (outputs exist) decide if job outputs exist or not if (job outputs do not exist) { if (attempt > 1) report job failed if (attempt > max) report failed, caExit() report starting an attempt emitStage(check, $attempt) buildHTML() submitOrRun() return # if not on grid, we need to call again to decide if job outputs exist } finishStage: report job finished successfully do any processing to make 'outputs exist' true above setGlobal(iteration, 0) emitStage() buildHTML() stopAfter() allDone: anything that runs after EVERY call canu-1.3/src/pipelines/canu/Gatekeeper.pm000066400000000000000000000374131272335422300204350ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/Gatekeeper.pm # # Modifications by: # # Brian P. Walenz from 2015-FEB-27 to 2015-SEP-21 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-27 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2016-FEB-29 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Gatekeeper; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(getMaxReadInStore getNumberOfReadsInStore getNumberOfBasesInStore getExpectedCoverage sequenceFileExists gatekeeper); use strict; use canu::Defaults; use canu::Execution; use canu::HTML; sub storeExists ($$) { my $wrk = shift @_; my $asm = shift @_; return (-e "$wrk/$asm.gkpStore"); } sub getMaxReadInStore ($$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $nr = 0; # No file, no reads. return($nr) if (! -e "$wrk/$asm.gkpStore/readlengthhistogram.txt"); # Read the info file. gatekeeperCreate creates this at the end. open(F, "< $wrk/$asm.gkpStore/readlengthhistogram.txt") or caExit("can't open '$wrk/$asm.gkpStore/readlengthhistogram.txt' for reading: $!", undef); while () { my @v = split '\s+', $_; $nr = $v[1]; } close(F); return($nr); } sub getNumberOfReadsInStore ($$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $nr = 0; # No file, no reads. return($nr) if (! -e "$wrk/$asm.gkpStore/info.txt"); # Read the info file. gatekeeperCreate creates this at the end. open(F, "< $wrk/$asm.gkpStore/info.txt") or caExit("can't open '$wrk/$asm.gkpStore/info.txt' for reading: $!", undef); while () { if (m/numReads\s+=\s+(\d+)/) { $nr = $1; } } close(F); return($nr); } sub getNumberOfBasesInStore ($$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $nb = 0; my $bin = getBinDirectory(); # No file, no bases. return($nb) if (! -e "$wrk/$asm.gkpStore/info.txt"); # Read the info file. gatekeeperCreate creates this at the end. open(F, "< $wrk/$asm.gkpStore/reads.txt") or caExit("can't open '$wrk/$asm.gkpStore/reads.txt' for reading: $!", undef); while () { my @v = split '\s+', $_; $nb += $v[2]; } close(F); return($nb); } sub getExpectedCoverage ($$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; return(int(getNumberOfBasesInStore($wrk, $asm) / getGlobal("genomeSize"))); } # Returns undef if a sequence file with the supplied name cannot be found. Common suffices and compressions are tried. # Otherwise, returns the found sequence file. sub sequenceFileExists ($) { my $p = shift @_; foreach my $s ("", ".fasta", ".fastq", ".fa", ".fq") { foreach my $c ("", ".gz", ".xz") { return("$p$s$c") if (-e "$p$s$c"); } } return(undef); } sub gatekeeperCreateStore ($$$@) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $tag = shift @_; my $bin = getBinDirectory(); my @inputs = @_; # If the store failed to build because of input errors and warnings, rename the store and continue. if (-e "$wrk/$asm.gkpStore.ACCEPTED") { rename("$wrk/$asm.gkpStore.ACCEPTED", "$wrk/$asm.gkpStore"); rename "$wrk/$asm.gkpStore.BUILDING.errorLog", "$wrk/$asm.gkpStore.errorLog"; return; } # Fail if there are no inputs. caExit("no input files specified, and store not already created, I have nothing to work on!", undef) if (scalar(@inputs) == 0); # Make sure all the inputs are here. my $failedFiles = undef; foreach my $iii (@inputs) { my $file = $iii; # This stupid foreach works by reference! $file = $2 if ($file =~ m/^(.*)\0(.*)/); # Handle the raw sequence inputs. if (! -e $file) { if (defined($failedFiles)) { $failedFiles .= "; '$file' not found in gatekeeper()"; } else { $failedFiles = "'$file' not found in gatekeeper()"; } } } caExit($failedFiles, undef) if defined($failedFiles); # Build a gkp file for all the raw sequence inputs. For simplicity, we just copy in any gkp # files as is. This documents what gatekeeper was built with, etc. open(F, "> $wrk/$asm.gkpStore.gkp") or caExit("cant' open '$wrk/$asm.gkpStore.gkp' for writing: $0", undef); foreach my $iii (@inputs) { if ($iii =~ m/^-(.*)\0(.*)$/) { my $tech = $1; my $file = $2; my @name = split '/', $2; my $name = $name[scalar(@name)-1]; $name = $1 if ($name =~ m/(.*).[xgb][z]2{0,1}$/i); $name = $1 if ($name =~ m/(.*).fast[aq]$/i); $name = $1 if ($name =~ m/(.*).f[aq]$/i); print F "########################################\n"; print F "# $tech: $file\n"; print F "#\n"; print F "name $name\n"; print F "preset $tech\n"; print F "$file\n"; print F "\n"; } elsif (-e $iii) { print F "########################################\n"; print F "# $iii\n"; print F "#\n"; open(I, "< $iii") or caExit("can't open gatekeeper input '$iii' for reading: $0", undef); while () { print F $_; } close(I); print F "\n"; } else { caExit("unrecognized gatekeeper input file '$iii'", undef); } } close(F); # Load the store. my $cmd; $cmd .= "$bin/gatekeeperCreate \\\n"; $cmd .= " -minlength " . getGlobal("minReadLength") . " \\\n"; $cmd .= " -o $wrk/$asm.gkpStore.BUILDING \\\n"; $cmd .= " $wrk/$asm.gkpStore.gkp \\\n"; $cmd .= "> $wrk/$asm.gkpStore.BUILDING.err 2>&1"; stopBefore("gatekeeper", $cmd); # A little funny business to make gatekeeper not fail on read quality issues. # A return code of 0 is total success. # A return code of 1 means it found errors in the inputs, but finished. # Anything larger is a crash. if (runCommand($wrk, $cmd) > 1) { caExit("gatekeeper failed", "$wrk/$asm.gkpStore.BUILDING.err"); } # Check for quality issues. if (-e "$wrk/$asm.gkpStore.BUILDING.err") { my $nProblems = 0; open(F, "< $wrk/$asm.gkpStore.BUILDING.err"); while () { $nProblems++ if (m/Check\syour\sreads/); } close(F); if ($nProblems > 0) { print STDERR "Gatekeeper detected problems in your input reads. Please review the logging in files:\n"; print STDERR " $wrk/$asm.gkpStore.BUILDING.err\n" if (getGlobal("stopOnReadQuality") == 0); print STDERR " $wrk/$asm.gkpStore.BUILDING.errorLog\n" if (getGlobal("stopOnReadQuality") == 0); print STDERR " $wrk/$asm.gkpStore.err\n" if (getGlobal("stopOnReadQuality") == 1); print STDERR " $wrk/$asm.gkpStore.errorLog\n" if (getGlobal("stopOnReadQuality") == 1); if (getGlobal("stopOnReadQuality")) { print STDERR "If you wish to proceed, rename the store with the following commands and restart canu.\n"; print STDERR "\n"; print STDERR " mv $wrk/$asm.gkpStore.BUILDING \\\n"; print STDERR " $wrk/$asm.gkpStore.ACCEPTED\n"; print STDERR "\n"; print STDERR "Or remove $wrk and re-run with stopOnReadQuality=false\n"; print STDERR "\n"; exit(1); } else { print STDERR "Proceeding with assembly because stopOnReadQuality=false.\n"; } } } rename "$wrk/$asm.gkpStore.BUILDING", "$wrk/$asm.gkpStore"; rename "$wrk/$asm.gkpStore.BUILDING.err", "$wrk/$asm.gkpStore.err"; rename "$wrk/$asm.gkpStore.BUILDING.errorLog", "$wrk/$asm.gkpStore.errorLog"; } sub gatekeeperGenerateReadsList ($$$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $tag = shift @_; my $bin = getBinDirectory(); if (runCommandSilently("$wrk/$asm.gkpStore", "$bin/gatekeeperDumpMetaData -G . -reads > reads.txt 2> /dev/null", 1)) { caExit("failed to generate list of reads in store", undef); } } sub gatekeeperGenerateLibrariesList ($$$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $tag = shift @_; my $bin = getBinDirectory(); if (runCommandSilently("$wrk/$asm.gkpStore", "$bin/gatekeeperDumpMetaData -G . -libs > libraries.txt 2> /dev/null", 1)) { caExit("failed to generate list of libraries in store", undef); } } sub gatekeeperGenerateReadLengths ($$$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $tag = shift @_; my $bin = getBinDirectory(); my $nb = 0; my @rl; my @hi; my $mm; open(F, "< $wrk/$asm.gkpStore/reads.txt") or caExit("can't open '$wrk/$asm.gkpStore/reads.txt' for reading: $!", undef); while () { my @v = split '\s+', $_; push @rl, $v[2]; # Save the length $nb += $v[2]; # Sum the bases $hi[int($v[2] / 1000)]++; # Add to the histogram (int truncates) } close(F); @rl = sort { $a <=> $b } @rl; $mm = int($rl[scalar(@rl)-1] / 1000); # max histogram value open(F, "> $wrk/$asm.gkpStore/readlengths.txt") or caExit("can't open '$wrk/$asm.gkpStore/readlengths.txt' for writing: $!", undef); foreach my $rl (@rl) { print F "$rl\n"; } close(F); open(F, "> $wrk/$asm.gkpStore/readlengthhistogram.txt") or caExit("can't open '$wrk/$asm.gkpStore/readlengthhistogram.txt' for writing: $!", undef); for (my $ii=0; $ii<=$mm; $ii++) { my $s = $ii * 1000; my $e = $ii * 1000 + 999; $hi[$ii] += 0; # Otherwise, cells with no count print as null. print F "$s\t$e\t$hi[$ii]\n"; } close(F); } sub gatekeeperGenerateReadLengthPlot ($$$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $tag = shift @_; my $bin = getBinDirectory(); open(F, "> $wrk/$asm.gkpStore/readlengths.gp") or caExit("can't open '$wrk/$asm.gkpStore/readlengths.gp' for writing: $!", undef); print F "set title 'read length'\n"; print F "set xlabel 'read length, bin width = 250'\n"; print F "set ylabel 'number of reads'\n"; print F "\n"; print F "binwidth=250\n"; print F "set boxwidth binwidth\n"; print F "bin(x,width) = width*floor(x/width) + binwidth/2.0\n"; print F "\n"; print F "set terminal png size 1024,1024\n"; print F "set output '$wrk/$asm.gkpStore/readlengths.lg.png'\n"; print F "plot [] '$wrk/$asm.gkpStore/readlengths.txt' using (bin(\$1,binwidth)):(1.0) smooth freq with boxes title ''\n"; print F "\n"; print F "set terminal png size 256,256\n"; print F "set output '$wrk/$asm.gkpStore/readlengths.sm.png'\n"; print F "plot [] '$wrk/$asm.gkpStore/readlengths.txt' using (bin(\$1,binwidth)):(1.0) smooth freq with boxes title ''\n"; close(F); if (runCommandSilently("$wrk/$asm.gkpStore", "gnuplot $wrk/$asm.gkpStore/readlengths.gp > /dev/null 2>&1", 0)) { print STDERR "--\n"; print STDERR "-- WARNING: gnuplot failed; no plots will appear in HTML output.\n"; print STDERR "--\n"; print STDERR "----------------------------------------\n"; } } sub gatekeeperReportReadLengthHistogram ($$$) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my $tag = shift @_; my $bin = getBinDirectory(); my $reads = getNumberOfReadsInStore($wrk, $asm); my $bases = getNumberOfBasesInStore($wrk, $asm); my $coverage = int(100 * $bases / getGlobal("genomeSize")) / 100; my $maxhist = 0; open(F, "< $wrk/$asm.gkpStore/readlengthhistogram.txt") or caExit("can't open '$wrk/$asm.gkpStore/readlengthhistogram.txt' for reading: $!", undef); while () { my @v = split '\s+', $_; $maxhist = ($maxhist < $v[2]) ? $v[2] : $maxhist; } close(F); my $scale = $maxhist / 70; print STDERR "--\n"; print STDERR "-- In gatekeeper store '$wrk/$asm.gkpStore':\n"; print STDERR "-- Found $reads reads.\n"; print STDERR "-- Found $bases bases ($coverage times coverage).\n"; print STDERR "--\n"; print STDERR "-- Read length histogram (one '*' equals ", int(100 * $scale) / 100, " reads):\n"; open(F, "< $wrk/$asm.gkpStore/readlengthhistogram.txt") or caExit("can't open '$wrk/$asm.gkpStore/readlengthhistogram.txt' for reading: $!", undef); while () { my @v = split '\s+', $_; printf STDERR "-- %6d %6d %6d %s\n", $v[0], $v[1], $v[2], "*" x int($v[2] / $scale); } close(F); } sub gatekeeper ($$$@) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $bin = getBinDirectory(); my @inputs = @_; $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); # An empty store? Remove it and try again. if ((storeExists($wrk, $asm)) && (getNumberOfReadsInStore($wrk, $asm) == 0)) { print STDERR "-- Removing empty or incomplate gkpStore '$wrk/$asm.gkpStore'\n"; runCommandSilently($wrk, "rm -rf $wrk/$asm.gkpStore", 1); } # Store with reads? Yay! Report it, then skip. goto allDone if (skipStage($WRK, $asm, "$tag-gatekeeper") == 1); goto allDone if (getNumberOfReadsInStore($wrk, $asm) > 0); gatekeeperCreateStore($wrk, $asm, $tag, @inputs) if (! -e "$wrk/$asm.gkpStore"); caExit("gatekeeper store exists, but contains no reads", undef) if (getNumberOfReadsInStore($wrk, $asm) == 0); gatekeeperGenerateReadsList($wrk, $asm, $tag) if (! -e "$wrk/$asm.gkpStore/reads.txt"); gatekeeperGenerateLibrariesList($wrk, $asm, $tag) if (! -e "$wrk/$asm.gkpStore/libraries.txt"); gatekeeperGenerateReadLengths($wrk, $asm, $tag) if (! -e "$wrk/$asm.gkpStore/readlengths.txt"); gatekeeperGenerateReadLengthPlot($wrk, $asm, $tag) if (! -e "$wrk/$asm.gkpStore/readlengths.png"); finishStage: gatekeeperReportReadLengthHistogram($wrk, $asm, $tag); emitStage($WRK, $asm, "$tag-gatekeeper"); buildHTML($WRK, $asm, $tag); stopAfter("gatekeeper"); allDone: } canu-1.3/src/pipelines/canu/Grid.pm000066400000000000000000000042471272335422300172450ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-NOV-27 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Grid; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(formatAllowedResources configureRemote); use strict; use canu::Defaults; # # Given a map of "cpu-mem" to number-of-nodes, write a log message, and return a string for use # later in the pipeline. # sub formatAllowedResources (\%$) { my $hosts_ref = shift @_; my $geName = shift @_; my %hosts = %$hosts_ref; my $hosts = ""; print STDERR "-- \n"; foreach my $c (keys %hosts) { my ($cpus, $mem) = split '-', $c; my $nodes = $hosts{$c}; printf(STDERR "-- Found %3d host%s with %3d core%s and %4d GB memory under $geName control.\n", $nodes, ($nodes == 1) ? " " : "s", $cpus, ($cpus == 1) ? " " : "s", $mem); $hosts .= "\0" if (defined($hosts)); $hosts .= "$cpus-$mem-$nodes"; } return $hosts; } sub configureRemote () { if ((getGlobal("useGrid") eq "remote") && (getGlobal("gridEngine") eq "")) { caExit("invalid 'useGrid=remote' specified; no gridEngine available", undef); } return if (uc(getGlobal("gridEngine")) ne ""); # Set to "1" so that shell scripts will emit "jobid=$1" at the start. setGlobalIfUndef("gridEngineTaskID", "1"); } canu-1.3/src/pipelines/canu/Grid_LSF.pm000066400000000000000000000126161272335422300177500ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-NOV-27 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-30 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Grid_LSF; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(detectLSF configureLSF); use strict; use canu::Defaults; use canu::Execution; use canu::Grid; sub detectLSF () { return if ( defined(getGlobal("gridEngine"))); my $bsub = findExecutable("bsub"); return if (!defined($bsub)); print STDERR "-- Detected LSF with 'bsub' binary in $bsub.\n"; setGlobal("gridEngine", "LSF"); } sub configureLSF () { return if (uc(getGlobal("gridEngine")) ne "LSF"); setGlobalIfUndef("gridEngineSubmitCommand", "bsub"); setGlobalIfUndef("gridEngineHoldOption", "-w \"ended\(\"WAIT_TAG\"\)\""); setGlobalIfUndef("gridEngineHoldOptionNoArray", "-w \"done\(\"WAIT_TAG\"\)\""); setGlobalIfUndef("gridEngineSyncOption", "-K"); setGlobalIfUndef("gridEngineNameOption", "-J"); setGlobalIfUndef("gridEngineArrayOption", ""); setGlobalIfUndef("gridEngineArrayName", "ARRAY_NAME\[ARRAY_JOBS\]"); setGlobalIfUndef("gridEngineArrayMaxJobs", 65535); setGlobalIfUndef("gridEngineOutputOption", "-o"); setGlobalIfUndef("gridEngineThreadsOption", "-n THREADS"); setGlobalIfUndef("gridEngineMemoryOption", "-M MEMORY"); setGlobalIfUndef("gridEnginePropagateCommand", "bmodify -w \"done\(\"WAIT_TAG\"\)\""); setGlobalIfUndef("gridEngineNameToJobIDCommand", "bjobs -A -J \"WAIT_TAG\" | grep -v JOBID"); setGlobalIfUndef("gridEngineNameToJobIDCommandNoArray", "bjobs -J \"WAIT_TAG\" | grep -v JOBID"); setGlobalIfUndef("gridEngineTaskID", "LSB_JOBINDEX"); setGlobalIfUndef("gridEngineArraySubmitID", "%I"); setGlobalIfUndef("gridEngineJobID", "LSB_JOBID"); # # LSF has variation in the units used to request memory # They are defined by the LSF_UNIT_FOR_LIMITS variable in lsf.conf # Poll and see if we can find it # my $memUnits = undef; open(F, "lsadmin showconf lim |"); my $s = ; # cluster name my $d = ; # dat/time while () { my @v = split '=', $_; if ($v[0] =~ m/LSF_UNIT_FOR_LIMITS/) { $memUnits = "t" if ($v[1] =~ m/[tT]/); $memUnits = "g" if ($v[1] =~ m/[gG]/); $memUnits = "m" if ($v[1] =~ m/[mM]/); $memUnits = "k" if ($v[1] =~ m/[kK]/); } } close(F); if (!defined($memUnits)) { print STDERR "-- Warning: unknown memory units for grid engine LSF assuming KB\n"; $memUnits = "k"; } # Build a list of the resources available in the grid. This will contain a list with keys # of "#CPUs-#GBs" and values of the number of nodes With such a config. Later on, we'll use this # to figure out what specific settings to use for each algorithm. # # The list is saved in global{"availableHosts"} # # !!! UNTESTED !! # my %hosts; open(F, "lshosts |"); my $h = ; # header my @h = split '\s+', $h; my $cpuIdx = 4; my $memIdx = 5; for (my $ii=0; ($ii < scalar(@h)); $ii++) { $cpuIdx = $ii if ($h[$ii] eq "ncpus"); $memIdx = $ii if ($h[$ii] eq "maxmem"); } while () { my @v = split '\s+', $_; my $cpus = $v[$cpuIdx]; my $mem = $v[$memIdx]; # if we failed to find the units from the configuration, inherit it from the lshosts output if (!defined($memUnits)) { $memUnits = "t" if ($mem =~ m/(\d+.*\d+)[tT]/); $memUnits = "g" if ($mem =~ m/(\d+.*\d+)[gG]/); $memUnits = "m" if ($mem =~ m/(\d+.*\d+)[mM]/); $memUnits = "k" if ($mem =~ m/(\d+.*\d+)[kK]/); } $mem = $1 * 1024 if ($mem =~ m/(\d+.*\d+)[tT]/); $mem = $1 * 1 if ($mem =~ m/(\d+.*\d+)[gG]/); $mem = $1 / 1024 if ($mem =~ m/(\d+.*\d+)[mM]/); $mem = $1 / 1024 / 1024 if ($mem =~ m/(\d+.*\d+)[kK]/); $mem = int($mem); $hosts{"$cpus-$mem"}++ if ($cpus gt 0); } close(F); setGlobal("availableHosts", formatAllowedResources(%hosts, "LSF")); setGlobal("gridEngineMemoryUnits", $memUnits); print STDERR "-- \n"; print STDERR "-- On LSF detected memory is requested in " . uc(${memUnits}) . "B\n"; print STDERR "-- \n"; } canu-1.3/src/pipelines/canu/Grid_PBSTorque.pm000066400000000000000000000103631272335422300211450ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-NOV-27 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-30 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Grid_PBSTorque; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(detectPBSTorque configurePBSTorque); use strict; use canu::Defaults; use canu::Execution; use canu::Grid; sub detectPBSTorque () { return if ( defined(getGlobal("gridEngine"))); my $pbsnodes = findExecutable("pbsnodes"); return if (!defined($pbsnodes)); print STDERR "-- Detected PBS/Torque with 'pbsnodes' binary in $pbsnodes.\n"; setGlobal("gridEngine", "PBS"); } sub configurePBSTorque () { return if (uc(getGlobal("gridEngine")) ne "PBS"); setGlobalIfUndef("gridEngineSubmitCommand", "qsub"); setGlobalIfUndef("gridEngineHoldOption", "-W depend=afteranyarray:WAIT_TAG"); setGlobalIfUndef("gridEngineHoldOptionNoArray", "-W depend=afterany:WAIT_TAG"); setGlobalIfUndef("gridEngineSyncOption", ""); setGlobalIfUndef("gridEngineNameOption", "-d `pwd` -N"); setGlobalIfUndef("gridEngineArrayOption", "-t ARRAY_JOBS"); setGlobalIfUndef("gridEngineArrayName", "ARRAY_NAME"); setGlobalIfUndef("gridEngineArrayMaxJobs", 65535); setGlobalIfUndef("gridEngineOutputOption", "-j oe -o"); setGlobalIfUndef("gridEngineThreadsOption", "-l nodes=1:ppn=THREADS"); setGlobalIfUndef("gridEngineMemoryOption", "-l mem=MEMORY"); setGlobalIfUndef("gridEnginePropagateCommand", "qalter -W depend=afterany:\"WAIT_TAG\""); setGlobalIfUndef("gridEngineNameToJobIDCommand", "qstat -f |grep -F -B 1 WAIT_TAG | grep Id: | grep -F [] |awk '{print \$NF}'"); setGlobalIfUndef("gridEngineNameToJobIDCommandNoArray", "qstat -f |grep -F -B 1 WAIT_TAG | grep Id: |awk '{print \$NF}'"); setGlobalIfUndef("gridEngineTaskID", "PBS_ARRAYID"); setGlobalIfUndef("gridEngineArraySubmitID", "\\\$PBS_ARRAYID"); setGlobalIfUndef("gridEngineJobID", "PBS_JOBID"); # Build a list of the resources available in the grid. This will contain a list with keys # of "#CPUs-#GBs" and values of the number of nodes With such a config. Later on, we'll use this # to figure out what specific settings to use for each algorithm. # # The list is saved in global{"availableHosts"} # my %hosts; open(F, "pbsnodes |"); while () { my $cpus = 0; my $mem = 0; if ($_ =~ m/status/) { my @stats = split ',', $_; for my $stat (@stats) { if ($stat =~ m/physmem/) { $mem = ( split '=', $stat )[-1]; } elsif ($stat =~ m/ncpus/) { $cpus = int(( split '=', $stat )[-1]); } } $mem = $1 * 1024 if ($mem =~ m/(\d+.*\d+)[tT]/); $mem = $1 * 1 if ($mem =~ m/(\d+.*\d+)[gG]/); $mem = $1 / 1024 if ($mem =~ m/(\d+.*\d+)[mM]/); $mem = $1 / 1024 / 1024 if ($mem =~ m/(\d+.*\d+)[kK]/); $mem = int($mem); $hosts{"$cpus-$mem"}++ if ($cpus gt 0); } } close(F); setGlobal("availableHosts", formatAllowedResources(%hosts, "PBS/Torque")); } canu-1.3/src/pipelines/canu/Grid_SGE.pm000066400000000000000000000261321272335422300177400ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-NOV-27 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-30 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Grid_SGE; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(detectSGE configureSGE); use strict; use canu::Defaults; use canu::Grid; use canu::Execution; sub detectSGE () { return if ( defined(getGlobal("gridEngine"))); return if (!defined($ENV{'SGE_ROOT'})); print STDERR "-- Detected Sun Grid Engine in '$ENV{'SGE_ROOT'}/$ENV{'SGE_CELL'}'.\n"; setGlobal("gridEngine", "SGE"); } sub configureSGE () { return if (uc(getGlobal("gridEngine")) ne "SGE"); my $maxArraySize = 65535; open(F, "qconf -sconf |") or caExit("can't run 'qconf' to get SGE config", undef); while () { if (m/max_aj_tasks\s+(\d+)/) { $maxArraySize = $1; } } close(F); setGlobalIfUndef("gridEngineSubmitCommand", "qsub"); setGlobalIfUndef("gridEngineHoldOption", "-hold_jid \"WAIT_TAG\""); setGlobalIfUndef("gridEngineHoldOptionNoArray", undef); setGlobalIfUndef("gridEngineSyncOption", "-sync y"); setGlobalIfUndef("gridEngineNameOption", "-cwd -N"); setGlobalIfUndef("gridEngineArrayOption", "-t ARRAY_JOBS"); setGlobalIfUndef("gridEngineArrayName", "ARRAY_NAME"); setGlobalIfUndef("gridEngineArrayMaxJobs", $maxArraySize); setGlobalIfUndef("gridEngineOutputOption", "-j y -o"); setGlobalIfUndef("gridEnginePropagateCommand", "qalter -hold_jid \"WAIT_TAG\""); setGlobalIfUndef("gridEngineThreadsOption", undef); #"-pe threads THREADS"); setGlobalIfUndef("gridEngineMemoryOption", undef); #"-l mem=MEMORY"); setGlobalIfUndef("gridEngineNameToJobIDCommand", undef); setGlobalIfUndef("gridEngineNameToJobIDCommandNoArray", undef); setGlobalIfUndef("gridEngineTaskID", "SGE_TASK_ID"); setGlobalIfUndef("gridEngineArraySubmitID", "\\\$TASK_ID"); setGlobalIfUndef("gridEngineJobID", "JOB_ID"); # Try to figure out the name of the threaded job execution environment. # It's the one with allocation_rule of $pe_slots. my $configError = 0; if (!defined(getGlobal("gridEngineThreadsOption"))) { my @env = `qconf -spl`; chomp @env; my @thr; my $bestThr = undef; my $bestSlots = 0; foreach my $env (@env) { my $ns = 0; my $ar = 0; my $jf = 0; open(F, "qconf -sp $env |"); while () { $ns = $1 if (m/slots\s+(\d+)/); # How many slots can we use? $ar = 1 if (m/allocation_rule.*pe_slots/); # All slots need to be on a single node. #$cs = 1 if (m/control_slaves.*FALSE/); # Doesn't apply to pe_slots. $jf = 1 if (m/job_is_first_task.*TRUE/); # The fisrt task (slot) does actual work. } close(F); next if ($ar == 0); next if ($jf == 0); push @thr, $env; if ($ns > $bestSlots) { $bestThr = $env; $bestSlots = $ns; } } if (scalar(@thr) == 1) { print STDERR "-- Detected Grid Engine environment '$thr[0]'.\n"; setGlobal("gridEngineThreadsOption", "-pe $thr[0] THREADS"); } elsif (scalar(@thr) > 1) { print STDERR "--\n"; print STDERR "-- WARNING: Couldn't determine the SGE parallel environment to run multi-threaded codes.\n"; print STDERR "-- WARNING: Valid choices are:\n"; foreach my $thr (@thr) { print STDERR "-- WARNING: gridEngineThreadsOption=\"-pe $thr THREADS\"\n"; } print STDERR "-- WARNING:\n"; print STDERR "-- WARNING: Using SGE parallel environment '$bestThr'.\n"; print STDERR "--\n"; setGlobal("gridEngineThreadsOption", "-pe $bestThr THREADS"); #$configError++; } else { print STDERR "--\n"; print STDERR "-- WARNING: Couldn't determine the SGE parallel environment to run multi-threaded codes.\n"; print STDERR "-- WARNING: No valid choices found! Find an appropriate Parallel Environment name (qconf -spl) and set:\n"; print STDERR "-- WARNING: gridEngineThreadsOption=\"-pe THREADS\"\n"; print STDERR "--\n"; $configError++; } } elsif (getGlobal("gridEngineThreadsOption") =~ m/-pe\s+(.*)\s+THREADS$/) { print STDERR "-- User supplied Grid Engine environment '", getGlobal("gridEngineThreadsOption"), "'.\n"; } else { caFailure("Couldn't parse gridEngineThreadsOption='" . getGlobal("gridEngineThreadsOption") . "'", undef); } # Try to figure out the name of the memory resource. if (!defined(getGlobal("gridEngineMemoryOption"))) { my @mem; open(F, "qconf -sc |"); while () { my @vals = split '\s+', $_; next if ($vals[5] ne "YES"); # Not a consumable resource. next if ($vals[2] ne "MEMORY"); # Not a memory resource. next if ($vals[0] =~ m/swap/); # Don't care about swap. next if ($vals[0] =~ m/virtual/); # Don't care about vm space. push @mem, $vals[0]; } close(F); if (scalar(@mem) == 1) { print STDERR "-- Detected Grid Engine consumable '$mem[0]'.\n"; setGlobal("gridEngineMemoryOption", "-l $mem[0]=MEMORY"); } elsif (scalar(@mem) > 1) { print STDERR "--\n"; print STDERR "-- WARNING: Couldn't determine the SGE resource to request memory.\n"; print STDERR "-- WARNING: Valid choices are (pick one and supply it to canu):\n"; foreach my $mem (@mem) { print STDERR "-- WARNING: gridEngineMemoryOption=\"-l $mem=MEMORY\"\n"; } print STDERR "--\n"; $configError++; } else { print STDERR "--\n"; print STDERR "-- WARNING: Couldn't determine the SGE resource to request memory.\n"; print STDERR "-- WARNING: No valid choices found! Find an appropriate complex name (qconf -sc) and set:\n"; print STDERR "-- WARNING: gridEngineMemoryOption=\"-l =MEMORY\"\n"; print STDERR "--\n"; $configError++; } } elsif (getGlobal("gridEngineMemoryOption") =~ m/^-l\s+.*=MEMORY$/) { print STDERR "-- User supplied Grid Engine consumable '", getGlobal("gridEngineMemoryOption"), "'.\n"; } else { caFailure("Couldn't parse gridEngineMemoryOption='" . getGlobal("gridEngineMemoryOption") . "'", undef); } caExit("can't configure for SGE", undef) if ($configError); # Check that SGE is setup to use the #! line instead of the (stupid) defaults. my %start_mode; my %start_shell; if (getGlobal('gridOptions') !~ m/-S/) { open(Q, "qconf -sql |"); while () { chomp; my $q = $_; $start_mode{$q} = "na"; $start_shell{$q} = "na"; open(F, "qconf -sq $q |"); while () { $start_mode{$q} = $1 if (m/shell_start_mode\s+(\S+)/); $start_shell{$q} = $1 if (m/shell\s+(\S+)/); } close(F); } my $startBad = undef; foreach my $q (keys %start_mode) { if (($start_mode{$q} ne "unix_behavior") && ($start_shell{$q} =~ m/csh$/)) { $startBad .= "-- WARNING: Queue '$q' has start mode set to 'posix_behavior' and shell set to '$start_shell{$q}'.\n"; } } if (defined($startBad)) { my $bash = findCommand("bash"); my $sh = findCommand("sh"); my $shell; $shell = $bash if ($bash ne ""); $shell = $sh if ($sh ne ""); print STDERR "--\n"; print STDERR "-- WARNING:\n"; print STDERR "$startBad"; print STDERR "-- WARNING:\n"; print STDERR "-- WARNING: Some queues in your configuration will fail to start jobs correctly.\n"; print STDERR "-- WARNING: Jobs will be submitted with option:\n"; print STDERR "-- WARNING: gridOptions=-S $shell\n"; print STDERR "-- WARNING:\n"; print STDERR "-- WARNING: If jobs fail to start, modify the above option to use a valid shell\n"; print STDERR "-- WARNING: and supply it directly to canu.\n"; print STDERR "-- WARNING:\n"; if (!defined(getGlobal('gridOptions'))) { setGlobal('gridOptions', "-S $shell"); } else { setGlobal('gridOptions', getGlobal('gridOptions') . " -S $shell"); } } } # Build a list of the resources available in the grid. This will contain a list with keys # of "#CPUs-#GBs" and values of the number of nodes With such a config. Later on, we'll use this # to figure out what specific settings to use for each algorithm. # # The list is saved in global{"availableHosts"} my %hosts; my $hosts = ""; open(F, "qhost |"); my $h = ; # Header my $b = ; # Table bar my @h = split '\s+', $h; my $cpuIdx = 2; my $memIdx = 4; for (my $ii=0; ($ii < scalar(@h)); $ii++) { $cpuIdx = $ii if ($h[$ii] eq "NCPU"); $memIdx = $ii if ($h[$ii] eq "MEMTOT"); } while () { my @v = split '\s+', $_; next if ($v[3] eq "-"); # Node disabled or otherwise not available my $cpus = $v[$cpuIdx]; my $mem = $v[$memIdx]; $mem = $1 * 1024 if ($mem =~ m/(\d+.*\d+)[tT]/); $mem = $1 * 1 if ($mem =~ m/(\d+.*\d+)[gG]/); $mem = $1 / 1024 if ($mem =~ m/(\d+.*\d+)[mM]/); $mem = int($mem); $hosts{"$cpus-$mem"}++ if ($cpus gt 0); } close(F); setGlobal("availableHosts", formatAllowedResources(%hosts, "Sun Grid Engine")); } canu-1.3/src/pipelines/canu/Grid_Slurm.pm000066400000000000000000000111521272335422300204200ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-NOV-27 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-30 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Grid_Slurm; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(detectSlurm configureSlurm); use strict; use canu::Defaults; use canu::Execution; use canu::Grid; sub detectSlurm () { return if ( defined(getGlobal("gridEngine"))); my $sinfo = findExecutable("sinfo"); return if (!defined($sinfo)); print STDERR "-- Detected Slurm with 'sinfo' binary in $sinfo.\n"; setGlobal("gridEngine", "SLURM"); } sub configureSlurm () { return if (uc(getGlobal("gridEngine")) ne "SLURM"); my $maxArraySize = 65535; open(F, "scontrol show config |") or caExit("can't run 'scontrol' to get SLURM config", undef); while () { if (m/MaxArraySize\s+=\s+(\d+)/) { $maxArraySize = $1; } } close(F); setGlobalIfUndef("gridEngineSubmitCommand", "sbatch"); setGlobalIfUndef("gridEngineHoldOption", "--depend=afterany:WAIT_TAG"); setGlobalIfUndef("gridEngineHoldOptionNoArray", "--depend=afterany:WAIT_TAG"); setGlobalIfUndef("gridEngineSyncOption", ""); ## TODO: SLURM may not support w/out wrapper; See LSF bsub manpage to compare setGlobalIfUndef("gridEngineNameOption", "-D `pwd` -J"); setGlobalIfUndef("gridEngineArrayOption", "-a ARRAY_JOBS"); setGlobalIfUndef("gridEngineArrayName", "ARRAY_NAME"); setGlobalIfUndef("gridEngineArrayMaxJobs", $maxArraySize); setGlobalIfUndef("gridEngineOutputOption", "-o"); ## NB: SLURM default joins STDERR & STDOUT if no -e specified setGlobalIfUndef("gridEngineThreadsOption", "--cpus-per-task=THREADS"); setGlobalIfUndef("gridEngineMemoryOption", "--mem-per-cpu=MEMORY"); setGlobalIfUndef("gridEnginePropagateCommand", "scontrol update job=\"WAIT_TAG\""); ## TODO: manually verify this in all cases setGlobalIfUndef("gridEngineNameToJobIDCommand", "squeue -h -o\%F -n \"WAIT_TAG\" | uniq"); ## TODO: manually verify this in all cases setGlobalIfUndef("gridEngineNameToJobIDCommandNoArray", "squeue -h -o\%i -n \"WAIT_TAG\""); ## TODO: manually verify this in all cases setGlobalIfUndef("gridEngineTaskID", "SLURM_ARRAY_TASK_ID"); setGlobalIfUndef("gridEngineArraySubmitID", "%A_%a"); setGlobalIfUndef("gridEngineJobID", "SLURM_JOB_ID"); # Build a list of the resources available in the grid. This will contain a list with keys # of "#CPUs-#GBs" and values of the number of nodes With such a config. Later on, we'll use this # to figure out what specific settings to use for each algorithm. # # The list is saved in global{"availableHosts"} # my %hosts; #NODELIST NODES CPUS MEMORY open(F, "sinfo --exact -o '%N %D %c %m' | grep -v drained | grep -v interactive |"); my $h = ; # header my @h = split '\s+', $h; my $nodeIdx = 1; my $cpuIdx = 4; my $memIdx = 6; for (my $ii=0; ($ii < scalar(@h)); $ii++) { $nodeIdx = $ii if ($h[$ii] eq "NODES"); $cpuIdx = $ii if ($h[$ii] eq "CPUS"); $memIdx = $ii if ($h[$ii] eq "MEMORY"); } while () { my @v = split '\s+', $_; my $cpus = $v[$cpuIdx]; my $mem = $v[$memIdx] / 1024; my $nodes = $v[$nodeIdx]; $hosts{"$cpus-$mem"}+=int($nodes) if ($cpus gt 0); } close(F); setGlobal("availableHosts", formatAllowedResources(%hosts, "Slurm")); } canu-1.3/src/pipelines/canu/HTML.pm000066400000000000000000000745431272335422300171320ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-NOV-08 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::HTML; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(buildHTML); use strict; use File::Copy; use File::Path qw(make_path remove_tree); use canu::Defaults; use canu::Execution; sub copyFile ($$) { my $sPath = shift @_; # Path to source file. my $dPath = shift @_; # Path to destination file. if ((-e $sPath) && ((! -e $dPath) || ((-M $sPath) < (-M $dPath)))) { copy($sPath, $dPath); } } sub simpleFigure ($$$$) { my $body = shift @_; my $sImage = shift @_; my $dImage = shift @_; my $text = shift @_; # No image? Note so in the html. if ((! -e "$sImage.sm.png") && (! -e "$sImage.lg.png") && (! -e "$dImage.sm.png") && (! -e "$dImage.lg.png")) { push @$body, "

Image '$sImage' not found.

\n"; return; } # Copy the file to our files location. copyFile("$sImage.lg.png", "$dImage.lg.png"); copyFile("$sImage.sm.png", "$dImage.sm.png"); # Empty image? Note so in the html. if ((-z "$dImage.sm.png") || (-z "$dImage.lg.png")) { push @$body, "

Image '$sImage' is empty. Probably no data to display.

\n"; return; } # Otherwise, show it! push @$body, "
\n"; push @$body, "\n"; push @$body, "
\n"; push @$body, "$text\n"; push @$body, "
\n"; push @$body, "
\n"; } sub buildGatekeeperHTML ($$$$$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $css = shift @_; # Array reference my $body = shift @_; # Array reference my $scripts = shift @_; # Array reference push @$body, "

Input Reads

\n"; push @$body, "\n"; if (! -e "$wrk/$asm.gkpStore/load.dat") { push @$body, "

None loaded.

\n"; return; } push @$body, "\n"; open(F, "< $wrk/$asm.gkpStore/load.dat") or caExit("can't open '$wrk/$asm.gkpStore/load.dat' for reading: $!", undef); while () { # nam blocks show up once per file. if (m/^nam\s(\d+)\s(.*)$/) { my $idx = $1; my $file = $2; push @$body, "\n"; push @$scripts, "document.getElementById('gkpload$idx').onclick = toggleTable;\n"; push @$scripts, "document.getElementById('gkpload$idx').style = 'cursor: pointer;';\n"; } # lib blocks show up once per file, all paramters are on the same line elsif (m/^lib\s/) { my @libs = split '\s+', $_; my ($param, $np, $var, $val); $param = shift @libs; # Throw out the first 'lib' word. $np = scalar(@libs); $param = shift @libs; # First thing we want to report. # First row needs to have a spanning cell for the 'parameters'. ($var, $val) = split '=', $param; push @$body, "\n"; # Remaining rows just have var=val. foreach $param (@libs) { ($var, $val) = split '=', $param; push @$body, "\n"; } } # dat blocks show up once per file, and are the last block emitted for a file elsif (m/^dat\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)$/) { my $nLOADEDA = $1; my $bLOADEDA = $2; my $nSKIPPEDA = $3; my $bSKIPPEDA = $4; my $nLOADEDQ = $5; my $bLOADEDQ = $6; my $nSKIPPEDQ = $7; my $bSKIPPEDQ = $8; my $nWARNS = $9; push @$body, "\n",; push @$body, "\n"; push @$body, "\n"; push @$body, "\n"; my $nl = $nLOADEDA + $nLOADEDQ; my $bl = $bLOADEDA + $bLOADEDQ; my $ns = $nSKIPPEDA + $nSKIPPEDQ; my $bs = $bSKIPPEDA + $bSKIPPEDQ; push @$body, "\n"; } # the sum block shows up excatly once, a summary of all the reads loaded elsif (m/^sum\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)$/) { my $nLOADED = $1; my $bLOADED = $2; my $nSKIPPED = $3; my $bSKIPPED = $4; my $nWARNS = $5; push @$body, "
$file
Parameters$var = $val
$var = $val
FASTA$nLOADEDA reads ($bLOADEDA bp)
$nSKIPPEDA reads ($bSKIPPEDA bp) were short and not loaded
FASTQ$nLOADEDQ reads ($bLOADEDQ bp)
$nSKIPPEDQ reads ($bSKIPPEDQ bp) were short and not loaded
$nl reads ($bl bp) loaded, $ns reads ($bs bp) skipped, $nWARNS warnings
\n"; push @$body, "\n"; push @$body, "

Final Store

\n"; push @$body, "\n"; push @$body, "\n"; push @$body, "\n"; push @$body, "\n"; push @$body, "\n"; push @$body, "\n"; push @$body, "
$wrk/$asm.gkpStore
readsLoaded$nLOADED reads ($bLOADED bp)
readsSkipped$nSKIPPED reads ($bSKIPPED bp) (read was too short)
warnings$nWARNS warnings (invalid base or quality value)
\n"; } else { caExit("failed to read '$wrk/$asm.gkpStore/load.log': invalid format", undef); } } close(F); push @$body, "

Read Length Histogram

\n"; simpleFigure($body, "$wrk/$asm.gkpStore/readlengths", "$wrk.html.files/readlengths", ""); } sub buildMerylHTML ($$$$$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $css = shift @_; # Array reference my $body = shift @_; # Array reference my $scripts = shift @_; # Array reference push @$body, "

k-Mer Counts

\n"; push @$body, "\n"; if (! -d "$wrk/0-mercounts") { push @$body, "

Stage not computed. ($wrk/0-mercounts)

\n"; return; } my %merSizes; open(F, "ls $wrk/0-mercounts/ |") or caExit("can't find files in '$wrk/0-mercounts': $!", undef); while () { if (m/\.ms(\d+)\./) { $merSizes{$1}++; } } close(F); foreach my $ms (keys %merSizes) { my $numTotal = 0; my $numDistinct = 0; my $numUnique = 0; my $largest = 0; if (-e "$wrk/0-mercounts/$asm.ms$ms.histogram.info") { open(F, "< $wrk/0-mercounts/$asm.ms$ms.histogram.info") or caExit("can't open '$wrk/0-mercounts/$asm.ms$ms.histogram.info' for reading: $!", undef); while () { $numTotal = $1 if (m/Found\s(\d+)\s+mers./); $numDistinct = $1 if (m/Found\s(\d+)\s+distinct\smers./); $numUnique = $1 if (m/Found\s(\d+)\s+unique\smers./); $largest = $1 if (m/Largest\smercount\sis\s(\d+)/); } close(F); simpleFigure($body, "$wrk/0-mercounts/$asm.ms$ms.histogram", "$wrk.html.files/$asm.ms$ms.histogram", "Histogram for k=$ms with $numTotal mers, $numDistinct distinct mers and $numUnique single-copy mers. Largest count is $largest."); } elsif ((-e "$wrk/0-mercounts/$asm.ms$ms.ignore") && (-z "$wrk/0-mercounts/$asm.ms$ms.ignore")) { push @$body, "Threshold zdero. No mers reported.\n"; } elsif ((-e "$wrk/0-mercounts/$asm.ms$ms.fasta") && (-z "$wrk/0-mercounts/$asm.ms$ms.fasta")) { push @$body, "Threshold zdero. No mers reported.\n"; } else { push @$body, "Using frequent mers in \n"; } } } sub buildCorrectionHTML ($$$$$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $css = shift @_; # Array reference my $body = shift @_; # Array reference my $scripts = shift @_; # Array reference # Need to include the minimum original read length that is correctable # Summarizes filterCorrectionOverlaps outputs. push @$body, "

Overlap Filtering

\n"; push @$body, "\n"; if (-e "$wrk/2-correction/$asm.globalScores.stats") { my $rh; # 'row header', for labeling a set of rows with a common cell push @$body, "\n"; open(F, "< $wrk/2-correction/$asm.globalScores.stats") or caExit("can't open '$wrk/2-correction/$asm.globalScores.stats' for reading: $!", undef); while () { chomp; next if (m/^$/); # Skip blank lines. push @$body, "\n" if ($_ eq "PARAMETERS:"); push @$body, "\n" if ($_ eq "OVERLAPS:"); push @$body, "\n" if ($_ eq "READS:"); $rh = "" if ($_ eq "PARAMETERS:"); $rh = "" if ($_ eq "OVERLAPS:"); # Gets replaced by 'IGNORED' below. $rh = "" if ($_ eq "READS:"); $rh = "" if ($_ eq "IGNORED:"); $rh = "" if ($_ eq "FILTERED:"); $rh = "" if ($_ eq "EVIDENCE:"); $rh = "" if ($_ eq "TOTAL:"); if (m/^\s*(\d+\.*\d*)\s+\((.*)\)$/) { push @$body, "$rh\n"; $rh = undef; } } close(F); push @$body, "
PARAMETERS
OVERLAPS
READS
IgnoredFilteredEvidenceTotal
$1$2
\n"; } else { push @$body, "

Stage not computed or results file removed ($wrk/2-correction/$asm.globalScores.stats).

\n"; } push @$body, "

Read Correction

\n"; push @$body, "\n"; # Summarizes expensiveFilter() outputs - we want to get the 'corrected read length filter' numbers. # which should be the first set in the file. my $nReads = undef; my $nBasesIn = undef; my $nBasesOut = undef; if (-e "$wrk/2-correction/$asm.readsToCorrect.summary") { open(F, "< $wrk/2-correction/$asm.readsToCorrect.summary") or caExit("can't open '$wrk/2-correction/$asm.readsToCorrect.summary' for reading: $!", undef); while () { $nReads = $1 if ((m/nReads\s+(\d+)/) && (!defined($nReads))); $nBasesIn = $1 if ((m/nBasds\s+(\d+).*input/) && (!defined($nBasesIn))); $nBasesOut = $1 if ((m/nReads\s+(\d+).*output/) && (!defined($nBasesOut))); last if (m/^Raw\sreads/); } close(F); push @$body, "

Filter method: corFilter=expensive. Expect to correct $nReads reads with ${nBasesIn}bp to ${nBasesOut}bp.

\n"; } else { push @$body, "

Filter method: corFilter=quick.

\n"; } # $wrk/2-correction/$asm.readsToCorrect has 'readID', 'originalLength' and 'expectedCorrectedLength'. # $WRK/$asm.correctedReads.length has 'readID', 'pieceID', 'length'. # # Both files should be sorted by increasing ID, so a simple merge sufficies. if (-e "$wrk/2-correction/$asm.correction.summary") { my $rh; push @$body, "\n"; open(F, "< $wrk/2-correction/$asm.correction.summary") or caExit("can't open '$wrk/2-correction/$asm.correction.summary' for reading: $!", undef); while () { chomp; next if (m/^$/); # Skip blank lines. push @$body, "\n" if ($_ eq "CORRECTION INPUTS:"); push @$body, "\n" if ($_ eq "CORRECTION OUTPUTS:"); push @$body, "\n" if ($_ eq "PIECES PER READ:"); # Normal table lines. if (m/^\s*(\d+\.*\d*)\s+\((.*)\)$/) { push @$body, "$rh\n"; $rh = undef; } # Pieces per read histogram. if (m/^\s*(\d+)\s+pieces:\s+(\d+)$/) { push @$body, "$rh\n"; $rh = undef; } } close(F); push @$body, "
INPUTS
OUTPUTS
PIECES PER READ
$1$2
$1$2
\n"; } # Really should be a 'caption' on the 'pieces per read' table. push @$body, "

A single input read can be split into multiple output reads, or possibly not even output at all.

\n"; # Simple vs Expensive filter true/false positive simpleFigure($body, "$wrk/2-correction/$asm.estimate.original-x-corrected", "correction.html.files/$asm.estimate.original-x-corrected", "Scatter plot of the original read length (X axis) against the expected corrected read length (Y axis).\n" . "Colors show a comparison of the simple filter (which doesn't use overlaps) to the expensive filter (which does).\n" . "A large green triangle (false negatives) hints that there could be abnormally low quality regions in the reads.\n"); # Scatter plots of read lengths - they don't show much. # Original vs expected shown above. simpleFigure($body, "$wrk/2-correction/$asm.originalLength-vs-expectedLength", "correction.html.files/$asm.originalLength-vs-expectedLength", "Scatter plot of original vs expected read length. Shown in filter plot above."); simpleFigure($body, "$wrk/2-correction/$asm.originalLength-vs-correctedLength", "correction.html.files/$asm.originalLength-vs-correctedLength", "Scatter plot of original vs corrected read length."); simpleFigure($body, "$wrk/2-correction/$asm.expectedLength-vs-correctedLength", "correction.html.files/$asm.expectedLength-vs-correctedLength", "Scatter plot of expected vs corrected read length."); # Histogram - expected vs corrected lengths NEEDS TO SHOW NEGATIVES!? simpleFigure($body, "$wrk/2-correction/$asm.length-difference-histograms", "correction.html.files/$asm.length-difference-histograms", "Histogram of the difference between the expected and corrected read lengths.\n" . "Note that a negative difference means the corrected read is larger than expected.\n"); # Histogram - original, expected, corrected lengths simpleFigure($body, "$wrk/2-correction/$asm.length-histograms", "correction.html.files/$asm.length-histograms", "Histogram of original (red), expected (green) and actual corrected (blue) read lengths.\n"); } sub buildTrimmingHTML ($$$$$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $css = shift @_; # Array reference my $body = shift @_; # Array reference my $scripts = shift @_; # Array reference push @$body, "

Trimming

\n"; push @$body, "\n"; if (-e "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.stats") { my $rh; # 'row header', for labeling a set of rows with a common cell # Read once to make a paramters table. We could have embedded this in the loop below, but it's cleaner here. #push @$body, "\n"; #push @$body, "
\n"; # Read again for the statistics. push @$body, "\n"; open(F, "< $wrk/3-overlapbasedtrimming/$asm.1.trimReads.stats") or caExit("can't open '$wrk/3-overlapbasedtrimming/$asm.1.trimReads.stats' for reading: $!", undef); while () { chomp; next if (m/^$/); # Skip blank lines. push @$body, "\n" if ($_ eq "PARAMETERS:"); push @$body, "
PARAMETERS
\n" if ($_ eq "INPUT READS:"); # Start a new table because 'params' has only push @$body, "\n" if ($_ eq "INPUT READS:"); # 2 cols, but the rest have 3 push @$body, "\n" if ($_ eq "INPUT READS:"); push @$body, "\n" if ($_ eq "INPUT READS:"); push @$body, "\n" if ($_ eq "OUTPUT READS:"); push @$body, "\n" if ($_ eq "OUTPUT READS:"); push @$body, "\n" if ($_ eq "TRIMMING DETAILS:"); push @$body, "\n" if ($_ eq "TRIMMING DETAILS:"); # Normal stats line "number (text)" if (m/^\s*(\d+\.*\d*)\s+\((.*)\)$/) { push @$body, "$rh\n"; $rh = undef; } # Specific to trimming "number reads number bases (text)" if (m/^\s*(\d+\.*\d*)\s+reads\s+(\d+\.*\d*)\s+bases\s+\((.*)\)$/) { push @$body, "$rh\n"; $rh = undef; } } close(F); push @$body, "
INPUT READS
readsbases
OUTPUT READS
readsbases
TRIMMING DETAILS
readsbases
$1$2
$1$2$3
\n"; } else { push @$body, "

Stage not computed or results file removed ($wrk/3-overlapbasedtrimming/$asm.1.trimReads.stats).

\n"; } simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.inputDeletedReads", "trimming.html.files/$asm.1.trimReads.inputDeletedReads", ""); simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.inputNoTrimReads", "trimming.html.files/$asm.1.trimReads.inputNoTrimReads", ""); simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.inputReads", "trimming.html.files/$asm.1.trimReads.inputReads", ""); simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.outputDeletedReads", "trimming.html.files/$asm.1.trimReads.outputDeletedReads", ""); simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.outputNoOvlReads", "trimming.html.files/$asm.1.trimReads.outputNoOvlReads", ""); simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.outputTrimmedReads", "trimming.html.files/$asm.1.trimReads.outputTrimmedReads", ""); simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.outputUnchangedReads", "trimming.html.files/$asm.1.trimReads.outputUnchangedReads", ""); simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.trim3", "trimming.html.files/$asm.1.trimReads.trim3", ""); simpleFigure($body, "$wrk/3-overlapbasedtrimming/$asm.1.trimReads.trim5", "trimming.html.files/$asm.1.trimReads.trim5", ""); push @$body, "

Splitting

\n"; push @$body, "\n"; if (-e "$wrk/3-overlapbasedtrimming/$asm.2.splitReads.stats") { my $rh; # 'row header', for labeling a set of rows with a common cell # Read once to make a paramters table. We could have embedded this in the loop below, but it's cleaner here. #push @$body, "\n"; #push @$body, "
\n"; # Read again for the statistics. push @$body, "\n"; open(F, "< $wrk/3-overlapbasedtrimming/$asm.2.splitReads.stats") or caExit("can't open '$wrk/3-overlapbasedtrimming/$asm.2.splitReads.stats' for reading: $!", undef); while () { chomp; next if (m/^$/); # Skip blank lines. push @$body, "\n" if ($_ eq "PARAMETERS:"); push @$body, "
PARAMETERS
\n" if ($_ eq "INPUT READS:"); # Start a new table because 'params' has only push @$body, "\n" if ($_ eq "INPUT READS:"); # 2 cols, but the rest have 3 push @$body, "\n" if ($_ eq "INPUT READS:"); push @$body, "\n" if ($_ eq "INPUT READS:"); push @$body, "\n" if ($_ eq "PROCESSED:"); push @$body, "\n" if ($_ eq "PROCESSED:"); push @$body, "\n" if ($_ eq "READS WITH SIGNALS:"); push @$body, "\n" if ($_ eq "READS WITH SIGNALS:"); push @$body, "\n" if ($_ eq "SIGNALS:"); push @$body, "\n" if ($_ eq "SIGNALS:"); push @$body, "\n" if ($_ eq "TRIMMING:"); push @$body, "\n" if ($_ eq "TRIMMING:"); # Normal stats line "number (text)" if (m/^\s*(\d+\.*\d*)\s+\((.*)\)$/) { push @$body, "$rh\n"; $rh = undef; } # Specific to trimming "number reads number bases (text)" if (m/^\s*(\d+\.*\d*)\s+reads\s+(\d+\.*\d*)\s+bases\s+\((.*)\)$/) { push @$body, "$rh\n"; $rh = undef; } if (m/^\s*(\d+\.*\d*)\s+reads\s+(\d+\.*\d*)\s+signals\s+\((.*)\)$/) { push @$body, "$rh\n"; $rh = undef; } } close(F); push @$body, "
INPUT READS
readsbases
PROCESSED
readsbases
READS WITH SIGNALS
readssignals
SIGNALS
readsbases
TRIMMING
readsbases
$1$2
$1$2$3
$1$2$3
\n"; } else { push @$body, "

Stage not computed or results file removed ($wrk/3-overlapbasedtrimming/$asm.2.splitReads.stats).

\n"; } #buildGatekeeperHTML($wrk, $asm, $tag, $css, $body, $scripts); # Analyzes the output fastq } sub buildOverlapperHTML ($$$$$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $css = shift @_; # Array reference my $body = shift @_; # Array reference my $scripts = shift @_; # Array reference push @$body, "

Overlaps

\n"; push @$body, "\n"; if (! -d "$wrk/$asm.ovlStore") { push @$body, "

Overlaps not computed.

\n"; return; } if (! -e "$wrk/$asm.ovlStore.summary") { push @$body, "

No statistics available for store '$wrk/$asm.ovlStore'.

\n"; return; } push @$body, "\n"; push @$body, "\n"; my ($category, $reads, $readsP, $length, $lengthsd, $size, $sizesd, $analysis); open(F, "< $wrk/$asm.ovlStore.summary") or caExit("Failed to open overlap store statistics in '$wrk/$asm.ovlStore': $!", undef); $_ = ; $_ = ; while () { chomp; next if ($_ eq ""); if (m/(.*)\s+(\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+\+-\s+(\d+.\d+)\s+(\d+.\d+)\s+\+-\s+(\d+.\d+)\s+\((.*)\)$/) { $category = $1; $reads = $2; $readsP = $3; $length = $4; $lengthsd = $5; $size = $6; $sizesd = $7; $analysis = $8; push @$body, "\n"; } elsif (m/(.*)\s+(\d+)\s+(\d+.\d+)\s+(\d+.\d+)\s+\+-\s+(\d+.\d+)\s+\((.*)\)$/) { $category = $1; $reads = $2; $readsP = $3; $length = $4; $lengthsd = $5; $size = undef; $sizesd = undef; $analysis = $6; push @$body, "\n"; } else { chomp; caExit("failed to parse line '$_' in file '$wrk/$asm.ovlStore.summary'", undef); } } close(F); push @$body, "
CategoryReads%Read LengthFeature Size or CoverageAnalysis
$category$reads$readsP$length±$lengthsd$size±$sizesd$analysis
$category$reads$readsP$length±$lengthsd$analysis
\n"; } sub buildOverlapErrorCorrectionHTML ($$$$$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $css = shift @_; # Array reference my $body = shift @_; # Array reference my $scripts = shift @_; # Array reference push @$body, "

Overlap Error Adjustment

\n"; push @$body, "\n"; } sub buildUnitiggerHTML ($$$$$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $css = shift @_; # Array reference my $body = shift @_; # Array reference my $scripts = shift @_; # Array reference push @$body, "

Unitigs

\n"; push @$body, "\n"; } sub buildConsensusHTML ($$$$$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $css = shift @_; # Array reference my $body = shift @_; # Array reference my $scripts = shift @_; # Array reference push @$body, "

Consensus

\n"; push @$body, "\n"; } sub buildOutputHTML ($$$$$$) { my $wrk = shift @_; my $asm = shift @_; my $tag = shift @_; my $css = shift @_; # Array reference my $body = shift @_; # Array reference my $scripts = shift @_; # Array reference push @$body, "

Final Outputs

\n"; push @$body, "\n"; } sub buildHTML ($$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my @css; my @body; my @scripts; $wrk = "$WRK/correction" if ($tag eq "cor"); $wrk = "$WRK/trimming" if ($tag eq "obt"); $wrk = "$WRK/unitigging" if ($tag eq "utg"); make_path("$wrk.html.files") if (! -e "$wrk.html.files"); # For correction runs if ($tag eq "cor") { push @body, "

Correction

\n"; buildGatekeeperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildMerylHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildOverlapperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildCorrectionHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); } # For trimming runs if ($tag eq "obt") { push @body, "

Trimming

\n"; buildGatekeeperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildMerylHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildOverlapperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildTrimmingHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); } # For assembly runs if ($tag eq "utg") { push @body, "

Assembly

\n"; buildGatekeeperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildMerylHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildOverlapperHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildOverlapErrorCorrectionHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildUnitiggerHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildConsensusHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); buildOutputHTML($wrk, $asm, $tag, \@css, \@body, \@scripts); } #print STDERR "WRITING '$wrk/$asm-summary.html'\n"; open(F, "> $wrk.html") or die "can't open '$wrk.html' for writing: $!\n"; print F "\n"; print F "\n"; print F "\n"; print F "\n"; print F "\n"; print F "canu analysis for assembly '$asm' in directory '$wrk'\n"; print F "\n"; print F "\n"; print F "\n"; print F "\n"; print F "\n"; print F @body; print F "\n"; print F "\n"; print F "\n"; print F "\n"; print F "\n"; print F "\n"; close(F); } canu-1.3/src/pipelines/canu/Meryl.pm000066400000000000000000000516321272335422300174500ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/Meryl.pm # # Modifications by: # # Brian P. Walenz from 2015-FEB-27 to 2015-AUG-25 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-NOV-03 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-19 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Meryl; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(getGenomeCoverage merylConfigure merylCheck merylProcess); use strict; use File::Path qw(make_path); use canu::Defaults; use canu::Execution; use canu::Gatekeeper; use canu::ErrorEstimate; use canu::HTML; sub getGenomeCoverage($$$) { my $wrk = shift @_; my $asm = shift @_; my $merSize = shift @_; my $bin = getBinDirectory(); my $gs=`cat $wrk/0-mercounts/$asm.ms$merSize.estMerThresh.err | grep "Guessed X coverage"|awk '{print \$NF}'`; chomp $gs; # if we couldn't find the coverage, just take # bases divided by user supplied genome size as a guestimate if ($gs == "") { open(F, "$bin/gatekeeperDumpMetaData -stats -G $wrk/$asm.gkpStore | ") or caFailure("failed to read gatekeeper stats fromfrom '$wrk/$asm.gkpStore'", undef); while () { my ($junk1, $library, $junk2, $reads, $junk3, $junk4, $bases, $junk5, $average, $junk6, $min, $junk7, $max) = split '\s+', $_; if ($library == 0) { $gs = $bases / getGlobal("genomeSize"); last; } } close(F); } return $gs; } # Generates # meryl mcdat/mcidx # mer histogram file # mer histogram plots # mer threshold # frequent mers # # Threshold: Three methods to pick it. # Threshold - 'auto', 'auto * X', 'auto / X', or an integer value # Distinct - by the fraction distinct retained # Total - by the fraction total retained # # We always compute canonical mers, that are not compressed. # Generates $wrk/0-mercounts/$asm.ms$merSize.frequentMers.fasta # stopBefore meryl (stops before meryl itself runs) # stopAfter meryl (stops after output is generated, even if it is just a symlink) sub plotHistogram ($$$$) { my $wrk = shift @_; # Local work directory my $ofile = shift @_; my $suffix = shift @_; my $size = shift @_; return if (-e "$ofile.histogram.$suffix.png"); open(F, "> $ofile.histogram.$suffix.gp"); print F "\n"; print F "unset multiplot\n"; print F "\n"; print F "set terminal png size $size,$size\n"; print F "set output '$ofile.histogram.$suffix.png'\n"; print F "\n"; print F "set multiplot\n"; print F "\n"; print F "# Distinct-vs-total full size plot\n"; print F "\n"; print F "set origin 0.0,0.0\n"; print F "set size 1.0,1.0\n"; print F "\n"; print F "set xrange [0.5:1.0]\n"; print F "set yrange [0.0:1.0]\n"; print F "\n"; print F "unset ytics\n"; print F "set y2tics 0.1\n"; #print F "set y2tics add ('0.6765' 0.6765)\n"; print F "\n"; print F "plot [0.5:1.0] '$ofile.histogram' using 3:4 with lines title 'Distinct-vs-Total'\n"; print F "\n"; print F "# Distinct-vs-total zoom in lower left corner\n"; print F "\n"; print F "set origin 0.05,0.10\n"; print F "set size 0.40,0.40\n"; print F "\n"; print F "set xrange [0.975:1.0]\n"; print F "set yrange [0.4:0.80]\n"; print F "\n"; print F "unset ytics\n"; # ytics on the left of the plot print F "set y2tics 0.1\n"; # y2tics on the right of the plot #print F "set y2tics add ('0.6765' 0.6765)\n"; print F "\n"; print F "plot [0.975:1.0] '$ofile.histogram' using 3:4 with lines title 'Distinct-vs-Total'\n"; print F "\n"; print F "# Histogram in upper left corner\n"; print F "\n"; print F "set origin 0.05,0.55\n"; print F "set size 0.40,0.40\n"; print F "\n"; print F "set xrange [0:200]\n"; print F "set yrange [0:30000000]\n"; print F "\n"; print F "unset ytics\n"; # ytics on the left of the plot print F "set y2tics 10e6\n"; # y2tics on the right of the plot print F "unset mytics\n"; print F "\n"; print F "plot [0:200] '$ofile.histogram' using 1:2 with lines title 'Histogram'\n"; close(F); if (runCommandSilently("$wrk/0-mercounts", "gnuplot $ofile.histogram.$suffix.gp > /dev/null 2>&1", 0)) { print STDERR "--\n"; print STDERR "-- WARNING: gnuplot failed; no plots will appear in HTML output.\n"; print STDERR "--\n"; print STDERR "----------------------------------------\n"; } } sub merylParameters ($$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) ## $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my ($wrk, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile); # Find a place to run stuff. $wrk = "$WRK/correction" if ($tag eq "cor"); $wrk = "$WRK/trimming" if ($tag eq "obt"); $wrk = "$WRK/unitigging" if ($tag eq "utg"); # Decide on which set of parameters we need to be using, and make output file names. if (getGlobal("${tag}Overlapper") eq "ovl") { $merSize = getGlobal("${tag}OvlMerSize"); $merThresh = getGlobal("${tag}OvlMerThreshold"); $merScale = 1.0; $merDistinct = getGlobal("${tag}OvlMerDistinct"); $merTotal = getGlobal("${tag}OvlMerTotal"); $ffile = "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.fasta"; # The fasta file we should be creating (ends in FASTA). $ofile = "$wrk/0-mercounts/$asm.ms$merSize"; # The meryl database 'intermediate file'. } elsif (getGlobal("${tag}Overlapper") eq "mhap") { $merSize = getGlobal("${tag}mhapMerSize"); $merThresh = undef; $merScale = 1.0; $merDistinct = undef; $merTotal = undef; $ffile = "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz"; # The mhap-specific file we should be creating (ends in IGNORE). $ofile = "$wrk/0-mercounts/$asm.ms$merSize"; # The meryl database 'intermediate file'. } elsif (getGlobal("${tag}Overlapper") eq "minimap") { # do nothing $ffile = "$wrk/0-mercounts/$asm.ms$merSize.skip"; make_path("$wrk/0-mercounts") if (! -d "$wrk/0-mercounts"); touch($ffile); } else { caFailure("unknown ${tag}Overlapper '" . getGlobal("${tag}Overlapper") . "'", undef); } # Decode the threshold. Auto with modifications ("auto * X") or ("auto / X")? if ($merThresh =~ m/auto\s*\*\s*(\S+)/) { $merThresh = "auto"; $merScale = $1; } if ($merThresh =~ m/auto\s*\/\s*(\S+)/) { $merThresh = "auto"; $merScale = 1.0 / $1; } # Return all this goodness. #print STDERR "wrk - '$wrk'\n"; #print STDERR "merSize - '$merSize'\n"; #print STDERR "merThresh - '$merThresh'\n"; #print STDERR "merScale - '$merScale'\n"; #print STDERR "merDistinct - '$merDistinct'\n"; #print STDERR "merTotal - '$merTotal'\n"; #print STDERR "ffile - '$ffile'\n"; #print STDERR "ofile - '$ofile'\n"; return($wrk, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile); } sub merylConfigure ($$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) ## $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $bin = getBinDirectory(); my $cmd; my ($wrk, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile) = merylParameters($WRK, $asm, $tag); goto allDone if (skipStage($WRK, $asm, "$tag-merylConfigure") == 1); goto allDone if (-e "$ffile"); goto allDone if ((-e "$ofile.mcidx") && (-e "$ofile.mcdat")); make_path("$wrk/0-mercounts") if (! -d "$wrk/0-mercounts"); # User supplied mers? Just symlink to them. if (defined(getGlobal("${tag}OvlFrequentMers"))) { #my $ffile = "$wrk/0-mercounts/$asm.frequentMers.fasta"; my $sfile = getGlobal("${tag}OvlFrequentMers"); if (! -e $ffile) { caFailure("${tag}OvlFrequentMers '$sfile' not found", undef) if (! -e $sfile); symlink $sfile, $ffile; } goto allDone; } # Nope, build a script. my $mem = int(getGlobal("merylMemory") * 1024 * 0.8); # Because meryl expects megabytes, not gigabytes. my $thr = getGlobal("merylThreads"); caExit("merylMemory isn't defined?", undef) if (!defined($mem)); caExit("merylThreads isn't defined?", undef) if (!defined($thr)); open(F, "> $wrk/0-mercounts/meryl.sh") or caExit("can't open '$wrk/0-mercounts/meryl.sh' for writing: $1", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "if [ -e $wrk/$asm.tigStore/seqDB.v001.tig ] ; then\n"; print F " exit 0\n"; print F "fi\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; print F "# Purge any previous intermediate result. Possibly not needed, but safer.\n"; print F "\n"; print F "rm -f $ofile.WORKING*\n"; print F "\n"; print F "\$bin/meryl \\\n"; print F " -B -C -L 2 -v -m $merSize -threads $thr -memory $mem \\\n"; print F " -s $wrk/$asm.gkpStore \\\n"; print F " -o $ofile.WORKING \\\n"; print F "&& \\\n"; print F "mv $ofile.WORKING.mcdat $ofile.FINISHED.mcdat \\\n"; print F "&& \\\n"; print F "mv $ofile.WORKING.mcidx $ofile.FINISHED.mcidx\n"; print F "\n"; print F "exit 0\n"; close(F); finishStage: emitStage($WRK, $asm, "merylConfigure"); buildHTML($WRK, $asm, $tag); stopAfter("merylConfigure"); allDone: } sub merylCheck ($$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) ## $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $attempt = getGlobal("canuIteration"); my $bin = getBinDirectory(); my $cmd; my ($wrk, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile) = merylParameters($WRK, $asm, $tag); # If the frequent mer file exists, don't bother running meryl. We don't really need the # databases. goto allDone if (skipStage($WRK, $asm, "$tag-meryl") == 1); goto allDone if (-e "$ffile"); goto allDone if ((-e "$ofile.mcidx") && (-e "$ofile.mcdat")); # If FINISHED exists, meryl finished successfully. If WORKING, it failed during output. If # nothing, it failed before output. if ((! -e "$ofile.FINISHED.mcdat") || (! -e "$ofile.FINISHED.mcidx")) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- meryl failed.\n"; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to generate mer counts. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- Meryl attempt $attempt begins.\n"; emitStage($WRK, $asm, "merylCheck", $attempt); buildHTML($WRK, $asm, $tag); submitOrRunParallelJob($WRK, $asm, "meryl", "$wrk/0-mercounts", "meryl", (1)); return; } finishStage: print STDERR "-- Meryl finished successfully.\n"; rename("$ofile.FINISHED.mcdat", "$ofile.mcdat"); rename("$ofile.FINISHED.mcidx", "$ofile.mcidx"); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "merylCheck"); buildHTML($WRK, $asm, $tag); stopAfter("merylCheck"); allDone: } sub merylProcess ($$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) ## $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $bin = getBinDirectory(); my $cmd; my ($wrk, $merSize, $merThresh, $merScale, $merDistinct, $merTotal, $ffile, $ofile) = merylParameters($WRK, $asm, $tag); goto allDone if (skipStage($WRK, $asm, "$tag-meryl") == 1); goto allDone if (-e "$ffile"); # A special case; if the threshold is zero, we can skip the rest. if ((defined($merThresh)) && ($merThresh ne "auto") && ($merThresh == 0) && (!defined($merDistinct)) && (!defined($merTotal))) { touch($ffile); goto allDone; } # Dump a histogram. if (! -e "$ofile.histogram") { $cmd = "$bin/meryl -Dh -s $ofile > $ofile.histogram 2> $ofile.histogram.info"; if (runCommand("$wrk/0-mercounts", $cmd)) { rename "$ofile.histogram", "$ofile.histogram.FAILED"; caFailure("meryl histogram failed", "$ofile.histogram.info"); } } # Compute a threshold, if needed if ($merThresh eq "auto") { if (! -e "$ofile.estMerThresh.out") { $cmd = "$bin/estimate-mer-threshold "; $cmd .= " -m $ofile "; $cmd .= " > $ofile.estMerThresh.out "; $cmd .= "2> $ofile.estMerThresh.err"; if (runCommand("$wrk/0-mercounts", $cmd)) { rename "$ofile.estMerThresh.out", "$ofile.estMerThresh.out.FAILED"; caFailure("estimate-mer-threshold failed", "$ofile.estMerThresh.err"); } } open(F, "< $ofile.estMerThresh.out") or caFailure("failed to read estimated mer threshold from '$ofile.estMerThresh.out'", undef); $merThresh = ; $merThresh = int($merThresh * $merScale); close(F); } # Compute a threshold based on the fraction distinct or total. if (defined($merDistinct) || defined($merTotal)) { open(F, "< $ofile.histogram") or caFailure("failed to read mer histogram from '$ofile.histogram'", undef); while () { my ($threshold, $num, $distinct, $total) = split '\s+', $_; if (($merThresh > 0) && ($merThresh < $threshold)) { print STDERR "-- Supplied merThreshold $merThresh is the smallest.\n"; last; } if ((defined($merDistinct)) && ($merDistinct <= $distinct)) { $merThresh = (($merThresh > 0) && ($merThresh < $threshold)) ? $merThresh : $threshold; print STDERR "-- Supplied merDistinct $merDistinct with threshold $threshold is the smallest.\n"; last; } if ((defined($merTotal)) && ($merTotal <= $total)) { $merThresh = (($merThresh > 0) && ($merThresh < $threshold)) ? $merThresh : $threshold; print STDERR "-- Supplied merTotal $merTotal with threshold $threshold is the smallest.\n"; last; } } close(F); } # Plot the histogram - annotated with the thesholds plotHistogram($wrk, $ofile, "lg", 1024); plotHistogram($wrk, $ofile, "sm", 256); # Generate the frequent mers for overlapper if ((getGlobal("${tag}Overlapper") eq "ovl") && (! -e $ffile)) { $cmd = "$bin/meryl -Dt -n $merThresh -s $ofile > $ffile 2> $ffile.err"; if (runCommand("$wrk/0-mercounts", $cmd)) { unlink $ffile; caFailure("meryl failed to dump frequent mers", "$ffile.err"); } unlink "$ffile.err"; } # Generate the frequent mers for mhap # # mer value numInstances totalKmers # TTTTGTTTTTTTTTTT 0.0000044602 589 132055862 # # The fraction is just $3/$4. I assume this is used with "--filter-threshold 0.000005". if ((getGlobal("${tag}Overlapper") eq "mhap") && (! -e $ffile)) { my $totalMers = 0; my $maxCount = 0; # Meryl reports number of distinct canonical mers, we multiply by two to get the # (approximate) number of distinct mers. Palindromes are counted twice, oh well. open(F, "< $ofile.histogram.info") or die "Failed to open '$ofile.histogram.info' for reading: $!\n"; while () { if (m/Found\s+(\d+)\s+mers./) { $totalMers = 2 * $1; } if (m/Largest\s+mercount\s+is\s+(\d+)./) { $maxCount = $1; } } close(F); caFailure("didn't find any mers?", "$ofile.histogram.info") if ($totalMers == 0); my $filterThreshold = (getGlobal("${tag}MhapSensitivity") eq "normal") ? getGlobal("${tag}MhapFilterThreshold") : getGlobal("${tag}MhapFilterThreshold"); # Also set in Meryl.pm my $misRate = 0.1; my $minCount = defined(getGlobal("${tag}MhapFilterUnique")) ? uniqueKmerThreshold($wrk, $asm, $merSize, $misRate)+1 : int($filterThreshold * $totalMers); my $totalToOutput = 0; my $totalFiltered = 0; open(F, "< $ofile.histogram") or die "Failed to open '$ofile.histogram' for reading: $!\n"; while () { my ($kCount, $occurences, $cumsum, $faction) = split '\s+', $_; if ($kCount < $minCount) { $totalFiltered = $cumsum * 100; } if ($kCount >= $minCount) { $totalToOutput += $occurences; } } close(F); $totalToOutput *= 2; # for the reverse complement open(F, "$bin/meryl -Dt -n $minCount -s $ofile | ") or die "Failed to run meryl to generate frequent mers $!\n"; open(O, "| gzip -c > $ofile.frequentMers.ignore.gz") or die "Failed to open '$ofile.frequentMers.ignore.gz' for writing: $!\n"; printf(O "%d\n", $totalToOutput); while (!eof(F)) { my $h = ; my $m = ; chomp $m; my $r = reverse $m; $r =~ tr/ACGTacgt/TGCAtgca/; if ($h =~ m/^>(\d+)/) { printf(O "%s\t%e\n", $m, $1 / $totalMers); printf(O "%s\t%e\n", $r, $1 / $totalMers); } } close(O); close(F); if (defined(getGlobal("${tag}MhapFilterUnique"))) { printf STDERR "-- For %s overlapping, filtering low-occurence k-mers < %d (%.2f\%) based on estimated error of %.2f\%.\n", getGlobal("${tag}Overlapper"), $minCount, $totalFiltered, 100*estimateRawError($wrk, $asm, $tag, $merSize); } printf STDERR "-- For %s overlapping, set repeat k-mer threshold to %d.\n", getGlobal("${tag}Overlapper"), int($filterThreshold * $totalMers); } # Report the new threshold. if ((getGlobal("${tag}Overlapper") eq "ovl") && ($merThresh > 0) && (getGlobal("${tag}OvlMerThreshold") ne $merThresh)) { print STDERR "-- Reset ${tag}OvlMerThreshold from ", getGlobal("${tag}OvlMerThreshold"), " to $merThresh.\n"; setGlobal("${tag}OvlMerThreshold", $merThresh); } finishStage: if (-e "$ofile.histogram.info") { my $numTotal = 0; my $numDistinct = 0; my $numUnique = 0; my $largest = 0; open(F, "< $ofile.histogram.info") or caFailure("can't open meryl histogram information file '$ofile.histogram.info' for reading: $!\n", undef); while () { $numTotal = $1 if (m/Found\s(\d+)\s+mers./); $numDistinct = $1 if (m/Found\s(\d+)\s+distinct\smers./); $numUnique = $1 if (m/Found\s(\d+)\s+unique\smers./); $largest = $1 if (m/Largest\smercount\sis\s(\d+)/); } close(F); print STDERR "--\n"; print STDERR "-- Found $numTotal $merSize-mers; $numDistinct distinct and $numUnique unique. Largest count $largest.\n"; } elsif (-z $ffile) { print STDERR "--\n"; print STDERR "-- Threshold zero. No mers will be to find.\n"; } else { print STDERR "--\n"; print STDERR "-- Using frequent mers in '", getGlobal("${tag}OvlFrequentMers"), "'\n"; } unlink "$ofile.mcidx" if (getGlobal("saveMerCounts") == 0); unlink "$ofile.mcdat" if (getGlobal("saveMerCounts") == 0); emitStage($WRK, $asm, "$tag-meryl"); buildHTML($WRK, $asm, $tag); stopAfter("meryl"); allDone: } canu-1.3/src/pipelines/canu/Output.pm000066400000000000000000000125731272335422300176610ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/Output.pm # # Modifications by: # # Brian P. Walenz from 2015-MAR-16 to 2015-AUG-25 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-NOV-02 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-DEC-02 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Output; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(outputLayout outputGraph outputSequence outputSummary); use strict; use POSIX qw(UINT_MAX); use canu::Defaults; use canu::Execution; use canu::HTML; # In this module, the inputs are read from $wrk (the local work directory), and the output are # written to $WRK (the root work directory). This is a change from CA8 where outputs # were written to the 9-terminator directory. sub outputLayout ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; goto allDone if (skipStage($WRK, $asm, "outputLayout") == 1); goto allDone if (-e "$WRK/$asm.layout"); $cmd = "$bin/tgStoreDump \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -T $wrk/$asm.tigStore 2 \\\n"; $cmd .= " -o $WRK/$asm \\\n"; $cmd .= " -layout \\\n"; $cmd .= "> $WRK/$asm.layout.err 2>&1"; if (runCommand($wrk, $cmd)) { caExit("failed to output layouts", "$WRK/$asm.layout.err"); } unlink "$WRK/$asm.layout.err"; finishStage: emitStage($WRK, $asm, "outputLayout"); buildHTML($WRK, $asm, "utg"); allDone: print STDERR "-- Unitig layouts saved in '$WRK/$asm.layout'.\n"; } sub outputGraph ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; goto allDone if (skipStage($WRK, $asm, "outputGraph") == 1); goto allDone if (-e "$WRK/$asm.gfa"); if (-e "$wrk/4-unitigger/$asm.unused.edges") { $cmd = "$bin/buildGraph \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -T $wrk/$asm.tigStore 2 \\\n"; $cmd .= " -E $wrk/4-unitigger/$asm.unused.edges \\\n"; $cmd .= " -o $WRK/$asm.gfa \\\n"; $cmd .= "2>&1 > $WRK/$asm.gfa.err\n"; if (runCommand($wrk, $cmd)) { caExit("failed to output consensus", "$WRK/$asm.gfa.err"); } unlink "$WRK/$asm.gfa.err"; } else { print STDERR "-- Unused best edges file missing, no graph output generated.\n"; } finishStage: emitStage($WRK, $asm, "outputGraph"); buildHTML($WRK, $asm, "utg"); allDone: print STDERR "-- Unitig graph saved in '$WRK/$asm.gfa'.\n"; } sub outputSequence ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $type = "fasta"; # Should probably be an option. goto allDone if (skipStage($WRK, $asm, "outputSequence") == 1); goto allDone if (-e "$WRK/$asm.contigs.$type"); foreach my $tt ("unassembled", "bubbles", "contigs") { $cmd = "$bin/tgStoreDump \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -T $wrk/$asm.tigStore 2 \\\n"; $cmd .= " -consensus -$type \\\n"; $cmd .= " -$tt \\\n"; $cmd .= "> $WRK/$asm.$tt.$type\n"; $cmd .= "2> $WRK/$asm.$tt.err"; if (runCommand($WRK, $cmd)) { caExit("failed to output consensus", "$WRK/$asm.$tt.err"); } unlink "$WRK/$asm.$tt.err"; } finishStage: emitStage($WRK, $asm, "outputSequence"); buildHTML($WRK, $asm, "utg"); allDone: print STDERR "-- Unitig sequences saved in '$WRK/$asm.*.$type'.\n"; } sub outputSummary ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; goto allDone if (skipStage($WRK, $asm, "outputSummary") == 1); goto allDone if (-e "$WRK/unitiggging.html"); finishStage: emitStage($WRK, $asm, "outputSummary"); buildHTML($WRK, $asm, "utg"); allDone: print STDERR "-- Summary saved in '$WRK/unitigging.html'.\n"; } canu-1.3/src/pipelines/canu/OverlapBasedTrimming.pm000066400000000000000000000161311272335422300224310ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/OverlapBasedTrimming.pm # # Modifications by: # # Brian P. Walenz from 2015-MAR-16 to 2015-AUG-25 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-NOV-04 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::OverlapBasedTrimming; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(qualTrimReads dedupeReads trimReads splitReads dumpReads); use strict; use File::Path qw(make_path remove_tree); use canu::Defaults; use canu::Execution; use canu::HTML; sub trimReads ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/trimming"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/3-overlapbasedtrimming"; goto allDone if (skipStage($WRK, $asm, "obt-trimReads") == 1); goto allDone if (-e "$path/trimmed"); make_path($path) if (! -d $path); # Previously, we'd pick the error rate used by unitigger. Now, we don't know unitigger here, # and require an obt specific error rate. $cmd = "$bin/trimReads \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -O $wrk/$asm.ovlStore \\\n"; $cmd .= " -Co $path/$asm.1.trimReads.clear \\\n"; $cmd .= " -e " . getGlobal("obtErrorRate") . " \\\n"; $cmd .= " -minlength " . getGlobal("minReadLength") . " \\\n"; #$cmd .= " -Cm $path/$asm.max.clear \\\n" if (-e "$path/$asm.max.clear"); $cmd .= " -ol " . getGlobal("trimReadsOverlap") . " \\\n"; $cmd .= " -oc " . getGlobal("trimReadsCoverage") . " \\\n"; $cmd .= " -o $path/$asm.1.trimReads \\\n"; $cmd .= "> $path/$asm.1.trimReads.err 2>&1"; stopBefore("trimReads", $cmd); if (runCommand($path, $cmd)) { caFailure("trimReads failed", "$path/$asm.1.trimReads.err"); } caFailure("trimReads finished, but no '$asm.1.trimReads.clear' output found", undef) if (! -e "$path/$asm.1.trimReads.clear"); unlink("$path/$asm.1.trimReads.err"); if (0) { $cmd = "$bin/gatekeeperDumpFASTQ \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -c $path/$asm.1.trimReads.clear \\\n"; $cmd .= " -o $path/$asm.1.trimReads.trimmed \\\n"; $cmd .= "> $path/$asm.1.trimReads.trimmed.err 2>&1"; if (runCommand($path, $cmd)) { caFailure("dumping trimmed reads failed", "$wrk/$asm.1.trimReads.trimmed.err"); } } finishStage: touch("$path/trimmed"); emitStage($WRK, $asm, "obt-trimReads"); buildHTML($WRK, $asm, "obt"); allDone: } sub splitReads ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/trimming"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/3-overlapbasedtrimming"; goto allDone if (skipStage($WRK, $asm, "obt-splitReads") == 1); goto allDone if (-e "$path/splitted"); # Splitted? make_path($path) if (! -d $path); my $erate = getGlobal("obtErrorRate"); # Was this historically #$cmd .= " -mininniepair 0 -minoverhanging 0 \\\n" if (getGlobal("doChimeraDetection") eq "aggressive"); $cmd = "$bin/splitReads \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -O $wrk/$asm.ovlStore \\\n"; $cmd .= " -Ci $path/$asm.1.trimReads.clear \\\n" if (-e "$path/$asm.1.trimReads.clear"); #$cmd .= " -Cm $path/$asm.max.clear \\\n" if (-e "$path/$asm.max.clear"); $cmd .= " -Co $path/$asm.2.splitReads.clear \\\n"; $cmd .= " -e $erate \\\n"; $cmd .= " -minlength " . getGlobal("minReadLength") . " \\\n"; $cmd .= " -o $path/$asm.2.splitReads \\\n"; $cmd .= "> $path/$asm.2.splitReads.err 2>&1"; stopBefore("splitReads", $cmd); if (runCommand($path, $cmd)) { caFailure("splitReads failed", "$path/$asm.2.splitReads.err"); } caFailure("splitReads finished, but no '$asm.2.splitReads.clear' output found", undef) if (! -e "$path/$asm.2.splitReads.clear"); unlink("$path/$asm.2.splitReads.err"); if (0) { $cmd = "$bin/gatekeeperDumpFASTQ \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -c $path/$asm.2.splitReads.clear \\\n"; $cmd .= " -o $path/$asm.2.splitReads.trimmed \\\n"; $cmd .= "> $path/$asm.2.splitReads.trimmed.err 2>&1"; if (runCommand($path, $cmd)) { caFailure("dumping trimmed reads failed", "$wrk/$asm.2.splitReads.trimmed.err"); } } finishStage: touch("$path/splitted", "Splitted? Is that even a word?"); emitStage($WRK, $asm, "obt-splitReads"); buildHTML($WRK, $asm, "obt"); allDone: } sub dumpReads ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/trimming"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/3-overlapbasedtrimming"; my $inp; goto allDone if (skipStage($WRK, $asm, "obt-dumpReads") == 1); goto allDone if (-e "$WRK/$asm.trimmedReads.fasta"); goto allDone if (-e "$WRK/$asm.trimmedReads.fasta.gz"); make_path($path) if (! -d $path); $inp = "$path/$asm.1.trimReads.clear" if (-e "$path/$asm.1.trimReads.clear"); $inp = "$path/$asm.2.splitReads.clear" if (-e "$path/$asm.2.splitReads.clear"); caFailure("dumping trimmed reads failed; no 'clear' input", "$WRK/$asm.trimmedReads.err") if (!defined($inp)); $cmd = "$bin/gatekeeperDumpFASTQ -fasta -nolibname \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -c $inp \\\n"; $cmd .= " -o $WRK/$asm.trimmedReads.gz \\\n"; $cmd .= "> $WRK/$asm.trimmedReads.err 2>&1"; if (runCommand($wrk, $cmd)) { caFailure("dumping trimmed reads failed", "$WRK/$asm.trimmedReads.err"); } unlink("$WRK/$asm.trimmedReads.err"); # Need gatekeeperDumpFASTQ to also write a gkp input file #touch("$wrk/$asm.trimmedReads.gkp"); finishStage: emitStage($WRK, $asm, "obt-dumpReads"); buildHTML($WRK, $asm, "obt"); allDone: print STDERR "--\n"; print STDERR "-- Trimmed reads saved in '$WRK/$asm.trimmedReads.fasta.gz'\n"; } canu-1.3/src/pipelines/canu/OverlapErrorAdjustment.pm000066400000000000000000000506541272335422300230440ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/OverlapErrorAdjustment.pm # # Modifications by: # # Brian P. Walenz from 2015-FEB-27 to 2015-SEP-21 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-NOV-03 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2016-MAR-27 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::OverlapErrorAdjustment; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(readErrorDetectionConfigure readErrorDetectionCheck overlapErrorAdjustmentConfigure overlapErrorAdjustmentCheck updateOverlapStore); use strict; use File::Path qw(make_path remove_tree); use canu::Defaults; use canu::Execution; use canu::Gatekeeper; use canu::HTML; # Hardcoded to use utgOvlErrorRate sub concatOutput ($@) { my $outName = shift @_; my @successJobs = @_; open(O, "> $outName"); binmode(O); foreach my $f (@successJobs) { open(F, "< $f"); binmode(F); my $buf; my $len = sysread(F, $buf, 1024 * 1024); while ($len > 0) { syswrite(O, $buf, $len); $len = sysread(F, $buf, 1024 * 1024); } close(F); } close(O); foreach my $f (@successJobs) { unlink $f; } } sub readErrorDetectionConfigure ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $path = "$wrk/3-overlapErrorAdjustment"; return if (getGlobal("enableOEA") == 0); goto allDone if (skipStage($wrk, $asm, "readErrorDetectionConfigure") == 1); goto allDone if (-e "$path/red.red"); goto allDone if (-e "$wrk/$asm.ovlStore/adjustedEvalues"); goto allDone if (-d "$wrk/$asm.tigStore"); make_path("$path") if (! -d "$path"); # RED uses 13 bytes/base plus 12 bytes/overlap + space for evidence reads. my @readLengths; my @numOlaps; #print STDERR "$bin/gatekeeperDumpMetaData -G $wrk/$asm.gkpStore -reads\n"; open(F, "$bin/gatekeeperDumpMetaData -G $wrk/$asm.gkpStore -reads |"); while () { my @v = split '\s+', $_; $readLengths[$v[0]] = $v[2]; } close(F); #print STDERR "$bin/ovStoreDump -G $wrk/$asm.gkpStore -O $wrk/$asm.ovlStore -d -counts\n"; open(F, "$bin/ovStoreDump -G $wrk/$asm.gkpStore -O $wrk/$asm.ovlStore -d -counts |"); while () { my @v = split '\s+', $_; $numOlaps[$v[0]] = $v[1]; } close(F); # Make an array of partitions, putting as many reads into each as will fit in the desired memory. my @bgn; my @end; my $nj = 0; #getAllowedResources("", "red"); my $maxID = getNumberOfReadsInStore($wrk, $asm); my $maxMem = getGlobal("redMemory") * 1024 * 1024 * 1024; my $maxReads = getGlobal("redBatchSize"); my $maxBases = getGlobal("redBatchLength"); print STDERR "\n"; print STDERR "Configure RED for ", getGlobal("redMemory"), "gb memory with batches of at most ", ($maxReads > 0) ? $maxReads : "(unlimited)", " reads and ", ($maxBases > 0) ? $maxBases : "(unlimited)", " bases.\n"; print STDERR "\n"; my $reads = 0; my $bases = 0; my $olaps = 0; my $coverage = getExpectedCoverage($wrk, $asm); push @bgn, 1; for (my $id = 1; $id <= $maxID; $id++) { $reads += 1; $bases += $readLengths[$id]; $olaps += $numOlaps[$id]; # Guess how much extra memory used for overlapping reads. Small genomes tend to load every read in the store, # large genomes ... load repeats + 2 * coverage * bases in reads (times 2 for overlaps off of each end) my $memory = (13 * $bases) + (12 * $olaps) + (2 * $bases * $coverage); if ((($maxMem > 0) && ($memory >= $maxMem * 0.75)) || # Allow 25% slop (10% is probably sufficient) (($maxReads > 0) && ($reads >= $maxReads)) || (($maxBases > 0) && ($bases >= $maxBases)) || (($id == $maxID - 1))) { push @end, $id; printf(STDERR "RED job %3u from read %9u to read %9u - %7.3f GB for %7u reads - %7.3f GB for %9u olaps - %7.3f GB for evidence\n", $nj + 1, $bgn[$nj], $end[$nj], $memory / 1024 / 1024 / 1024, $reads, 13 * $bases / 1024 / 1024 / 1024, $bases, 12 * $olaps / 1024 / 1024 / 1024, $olaps, 2 * $bases * $coverage / 1024 / 1024 / 1024); $nj++; $reads = 0; $bases = 0; $olaps = 0; push @bgn, $id + 1; # RED expects inclusive ranges. } } # Dump a script. my $batchSize = getGlobal("redBatchSize"); my $numThreads = getGlobal("redThreads"); my $numReads = getNumberOfReadsInStore($wrk, $asm); open(F, "> $path/red.sh") or caExit("can't open '$path/red.sh' for writing: $!", undef); print F "#!" . getGlobal("shell") . "\n\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; for (my $jj=1; $jj <= $nj; $jj++) { print F "if [ \$jobid = $jj ] ; then\n"; print F " minid=$bgn[$jj-1]\n"; print F " maxid=$end[$jj-1]\n"; print F "fi\n"; } print F "jobid=`printf %04d \$jobid`\n"; print F "\n"; print F "if [ -e $path/\$jobid.red ] ; then\n"; print F " echo Job previously completed successfully.\n"; print F " exit\n"; print F "fi\n"; print F getBinDirectoryShellCode(); print F "if [ ! -e $path/\$jobid.red ] ; then\n"; print F " \$bin/findErrors \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -O $wrk/$asm.ovlStore \\\n"; print F " -R \$minid \$maxid \\\n"; print F " -e " . getGlobal("utgOvlErrorRate") . " -l " . getGlobal("minOverlapLength") . " \\\n"; print F " -o $path/\$jobid.red.WORKING \\\n"; print F " -t $numThreads \\\n"; print F " && \\\n"; print F " mv $path/\$jobid.red.WORKING $path/\$jobid.red\n"; print F "fi\n"; close(F); chmod 0755, "$path/red.sh"; finishStage: emitStage($WRK, $asm, "readErrorDetectionConfigure"); buildHTML($WRK, $asm, "utg"); allDone: } sub readErrorDetectionCheck ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $attempt = getGlobal("canuIteration"); my $path = "$wrk/3-overlapErrorAdjustment"; return if (getGlobal("enableOEA") == 0); goto allDone if (skipStage($wrk, $asm, "readErrorDetectionCheck", $attempt) == 1); goto allDone if (-e "$path/red.red"); goto allDone if (-e "$wrk/$asm.ovlStore/adjustedEvalues"); goto allDone if (-d "$wrk/$asm.tigStore"); # Figure out if all the tasks finished correctly. my @successJobs; my @failedJobs; my $failureMessage = ""; open(A, "< $path/red.sh") or caExit("can't open '$path/red.sh' for reading: $!\n", undef); while () { if (m/if.*jobid\s+=\s+(\d+)\s+.*then/) { my $ji = substr("0000" . $1, -4); my $jn = "$path/$ji.red"; if (! -e $jn) { $failureMessage .= "-- job $jn FAILED.\n"; push @failedJobs, $1; } else { push @successJobs, $jn; } } } close(A); # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " read error detection jobs failed:\n"; print STDERR $failureMessage; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to detect errors in reads. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- read error detection attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "readErrorDetectionCheck", $attempt); buildHTML($WRK, $asm, "utg"); submitOrRunParallelJob($WRK, $asm, "red", "$path", "red", @failedJobs); return; } finishStage: print STDERR "-- Found ", scalar(@successJobs), " read error detection output files.\n"; # I didn't wan't to concat all the corrections, but it is _vastly_ easier to do so, compared to # hacking correctOverlaps to handle multiple corrections files. Plus, it is now really just a # concat; before, the files needed to be parsed to strip off a header. concatOutput("$path/red.red", @successJobs); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "readErrorDetectionCheck"); buildHTML($WRK, $asm, "utg"); stopAfter("red"); allDone: } sub overlapErrorAdjustmentConfigure ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $path = "$wrk/3-overlapErrorAdjustment"; return if (getGlobal("enableOEA") == 0); goto allDone if (skipStage($wrk, $asm, "overlapErrorAdjustmentConfigure") == 1); goto allDone if (-e "$path/oea.sh"); goto allDone if (-e "$wrk/$asm.ovlStore/adjustedEvalues"); goto allDone if (-d "$wrk/$asm.tigStore"); # OEA uses 1 byte/base + 8 bytes/adjustment + 28 bytes/overlap. We don't know the number of adjustments, but that's # basically error rate. No adjustment is output for mismatches. my @readLengths; my @numOlaps; #print STDERR "$bin/gatekeeperDumpMetaData -G $wrk/$asm.gkpStore -reads\n"; open(F, "$bin/gatekeeperDumpMetaData -G $wrk/$asm.gkpStore -reads |"); while () { my @v = split '\s+', $_; $readLengths[$v[0]] = $v[2]; } close(F); #print STDERR "$bin/ovStoreDump -G $wrk/$asm.gkpStore -O $wrk/$asm.ovlStore -d -counts\n"; open(F, "$bin/ovStoreDump -G $wrk/$asm.gkpStore -O $wrk/$asm.ovlStore -d -counts |"); while () { my @v = split '\s+', $_; $numOlaps[$v[0]] = $v[1]; } close(F); # Make an array of partitions, putting as many reads into each as will fit in the desired memory. my @bgn; my @end; my $nj = 0; #getAllowedResources("", "oea"); my $maxID = getNumberOfReadsInStore($wrk, $asm); my $maxMem = getGlobal("oeaMemory") * 1024 * 1024 * 1024; my $maxReads = getGlobal("oeaBatchSize"); my $maxBases = getGlobal("oeaBatchLength"); print STDERR "\n"; print STDERR "Configure OEA for ", getGlobal("oeaMemory"), "gb memory with batches of at most ", ($maxReads > 0) ? $maxReads : "(unlimited)", " reads and ", ($maxBases > 0) ? $maxBases : "(unlimited)", " bases.\n"; print STDERR "\n"; my $reads = 0; my $bases = 0; my $olaps = 0; my $coverage = getExpectedCoverage($wrk, $asm); my $corrSize = (-s "$path/red.red"); push @bgn, 1; for (my $id = 1; $id <= $maxID; $id++) { $reads += 1; $bases += $readLengths[$id]; $olaps += $numOlaps[$id]; # Hacked to attempt to estimate adjustment size better. Olaps should only require 12 bytes each. my $memBases = (1 * $bases); # Corrected reads for this batch my $memAdj1 = (8 * $corrSize) * 0.33; # Overestimate of the size of the indel adjustments needed (total size includes mismatches) my $memReads = (32 * $reads); # Read data in the batch my $memOlaps = (32 * $olaps); # Loaded overlaps my $memSeq = (4 * 2097152); # two char arrays of 2*maxReadLen my $memAdj2 = (16 * 2097152); # two Adjust_t arrays of maxReadLen my $memWA = (32 * 1048576); # Work area (16mb) and edit array (16mb) my $memMisc = (256 * 1048576); # Work area (16mb) and edit array (16mb) and (192mb) slop my $memory = $memBases + $memAdj1 + $memReads + $memOlaps + $memSeq + $memAdj2 + $memWA + $memMisc; if ((($maxMem > 0) && ($memory >= $maxMem * 0.75)) || (($maxReads > 0) && ($reads >= $maxReads)) || (($maxBases > 0) && ($bases >= $maxBases)) || (($id == $maxID))) { push @end, $id; printf(STDERR "OEA job %3u from read %9u to read %9u - %4.1f bases + %4.1f adjusts + %4.1f reads + %4.1f olaps + %4.1f fseq/rseq + %4.1f fadj/radj + %4.1f work + %4.1f misc = %5.1f MB\n", $nj + 1, $bgn[$nj], $end[$nj], $memBases / 1024 / 1024, $memAdj1 / 1024 / 1024, $memReads / 1024 / 1024, $memOlaps / 1024 / 1024, $memSeq / 1024 / 1024, $memAdj2 / 1024 / 1024, $memWA / 1024 / 1024, $memMisc / 1024 / 1024, $memory / 1024 / 1024); $nj++; $reads = 0; $bases = 0; $olaps = 0; push @bgn, $id + 1; # OEA expects inclusive ranges. } } # Dump a script open(F, "> $path/oea.sh") or caExit("can't open '$path/oea.sh' for writing: $!", undef); print F "#!" . getGlobal("shell") . "\n\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; for (my $jj=1; $jj <= $nj; $jj++) { print F "if [ \$jobid = $jj ] ; then\n"; print F " minid=$bgn[$jj-1]\n"; print F " maxid=$end[$jj-1]\n"; print F "fi\n"; } print F "jobid=`printf %04d \$jobid`\n"; print F "\n"; print F "if [ -e $path/\$jobid.oea ] ; then\n"; print F " echo Job previously completed successfully.\n"; print F " exit\n"; print F "fi\n"; print F getBinDirectoryShellCode(); print F "if [ ! -e $path/\$jobid.oea ] ; then\n"; print F " \$bin/correctOverlaps \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -O $wrk/$asm.ovlStore \\\n"; print F " -R \$minid \$maxid \\\n"; print F " -e " . getGlobal("utgOvlErrorRate") . " -l " . getGlobal("minOverlapLength") . " \\\n"; print F " -c $path/red.red \\\n"; print F " -o $path/\$jobid.oea.WORKING \\\n"; print F " && \\\n"; print F " mv $path/\$jobid.oea.WORKING $path/\$jobid.oea\n"; print F "fi\n"; close(F); chmod 0755, "$path/oea.sh"; finishStage: emitStage($WRK, $asm, "overlapErrorAdjustmentConfigure"); buildHTML($WRK, $asm, "utg"); allDone: } sub overlapErrorAdjustmentCheck ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $attempt = getGlobal("canuIteration"); my $path = "$wrk/3-overlapErrorAdjustment"; return if (getGlobal("enableOEA") == 0); goto allDone if (skipStage($wrk, $asm, "overlapErrorAdjustmentCheck", $attempt) == 1); goto allDone if (-e "$path/oea.files"); # Figure out if all the tasks finished correctly. my $batchSize = getGlobal("oeaBatchSize"); my $failedJobs = 0; my $numReads = getNumberOfReadsInStore($wrk, $asm); my $numJobs = 0; #int($numReads / $batchSize) + (($numReads % $batchSize == 0) ? 0 : 1); # Need to read script to find number of jobs! my @successJobs; my @failedJobs; my $failureMessage = ""; open(A, "< $path/oea.sh") or caExit("can't open '$path/oea.sh' for reading: $!\n", undef); while () { if (m/if.*jobid\s+=\s+(\d+)\s+.*then/) { my $ji = substr("0000" . $1, -4); my $jn = "$path/$ji.oea"; if (! -e $jn) { $failureMessage .= "-- job $jn FAILED.\n"; push @failedJobs, $1; } else { push @successJobs, $jn; } } } close(A); # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " overlap error adjustment jobs failed:\n"; print STDERR $failureMessage; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to adjust overlap error rates. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- overlap error adjustment attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "overlapErrorAdjustmentCheck", $attempt); buildHTML($WRK, $asm, "utg"); submitOrRunParallelJob($WRK, $asm, "oea", "$path", "oea", @failedJobs); return; } finishStage: print STDERR "-- Found ", scalar(@successJobs), " overlap error adjustment output files.\n"; open(L, "> $path/oea.files") or caExit("can't open '$path/oea.files' for writing: $!", undef); foreach my $f (@successJobs) { print L "$f\n"; } close(L); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "overlapErrorAdjustmentCheck"); buildHTML($WRK, $asm, "utg"); stopAfter("oea"); allDone: } sub updateOverlapStore ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; my $path = "$wrk/3-overlapErrorAdjustment"; return if (getGlobal("enableOEA") == 0); goto allDone if (skipStage($wrk, $asm, "updateOverlapStore") == 1); goto allDone if (-e "$wrk/$asm.ovlStore/evalues"); goto allDone if (-e "$wrk/$asm.ovlStore/adjustedEvalues"); goto allDone if (-d "$wrk/$asm.tigStore"); caExit("didn't find '$path/oea.files' to add to store, yet overlapper finished", undef) if (! -e "$path/oea.files"); $cmd = "$bin/ovStoreBuild \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -O $wrk/$asm.ovlStore \\\n"; $cmd .= " -evalues \\\n"; $cmd .= " -L $path/oea.files \\\n"; $cmd .= "> $path/oea.apply.err 2>&1"; if (runCommand("$path", $cmd)) { unlink "$wrk/$asm.ovlStore/evalues"; caExit("failed to add error rates to overlap store", "$path/oea.apply.err"); } finishStage: emitStage($WRK, $asm, "updateOverlapStore"); buildHTML($WRK, $asm, "utg"); allDone: } canu-1.3/src/pipelines/canu/OverlapInCore.pm000066400000000000000000000362521272335422300210710ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/OverlapInCore.pm # # Modifications by: # # Brian P. Walenz from 2015-FEB-27 to 2015-AUG-25 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-19 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::OverlapInCore; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(overlapConfigure overlap overlapCheck); use strict; use File::Path qw(make_path remove_tree); use canu::Defaults; use canu::Execution; use canu::HTML; sub overlapConfigure ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $type = shift @_; my $bin = getBinDirectory(); my $cmd; $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); my $path = "$wrk/1-overlapper"; caFailure("invalid type '$type'", undef) if (($type ne "partial") && ($type ne "normal")); goto allDone if (skipStage($WRK, $asm, "$tag-overlapConfigure") == 1); goto allDone if (-e "$path/$asm.partition.ovlopt"); goto allDone if (-e "$path/ovljob.files"); goto allDone if (-e "$wrk/$asm.ovlStore"); print STDERR "--\n"; print STDERR "-- OVERLAPPER (normal) (correction) erate=", getGlobal("${tag}OvlErrorRate"), "\n" if ($tag eq "cor"); print STDERR "-- OVERLAPPER (normal) (trimming) erate=", getGlobal("${tag}OvlErrorRate"), "\n" if ($tag eq "obt"); print STDERR "-- OVERLAPPER (normal) (assembly) erate=", getGlobal("${tag}OvlErrorRate"), "\n" if ($tag eq "utg"); print STDERR "--\n"; make_path("$path") if (! -d "$path"); if (! -e "$path/$asm.partition.ovlopt") { # These used to be runCA options, but were removed in canu. They were used mostly for illumina-pacbio correction, # but were also used (or could have been used) during the Salmon assembly when overlaps were computed differently # depending on the libraries involved (and was run manually). These are left in for documentation. # #my $checkLibrary = getGlobal("${tag}CheckLibrary"); #my $hashLibrary = getGlobal("${tag}HashLibrary"); #my $refLibrary = getGlobal("${tag}RefLibrary"); my $hashBlockLength = getGlobal("${tag}OvlHashBlockLength"); my $hashBlockSize = 0; my $refBlockSize = getGlobal("${tag}OvlRefBlockSize"); my $refBlockLength = getGlobal("${tag}OvlRefBlockLength"); my $minOlapLength = getGlobal("minOverlapLength"); if (($refBlockSize > 0) && ($refBlockLength > 0)) { caExit("can't set both ${tag}OvlRefBlockSize and ${tag}OvlRefBlockLength", undef); } $cmd = "$bin/overlapInCorePartition \\\n"; $cmd .= " -g $wrk/$asm.gkpStore \\\n"; $cmd .= " -bl $hashBlockLength \\\n"; $cmd .= " -bs $hashBlockSize \\\n"; $cmd .= " -rs $refBlockSize \\\n"; $cmd .= " -rl $refBlockLength \\\n"; #$cmd .= " -H $hashLibrary \\\n" if ($hashLibrary ne "0"); #$cmd .= " -R $refLibrary \\\n" if ($refLibrary ne "0"); #$cmd .= " -C \\\n" if (!$checkLibrary); $cmd .= " -ol $minOlapLength \\\n"; $cmd .= " -o $path/$asm.partition \\\n"; $cmd .= "> $path/$asm.partition.err 2>&1"; if (runCommand($wrk, $cmd)) { caExit("failed partition for overlapper", undef); } } open(BAT, "< $path/$asm.partition.ovlbat") or caExit("can't open '$path/$asm.partition.ovlbat' for reading: $!", undef); open(JOB, "< $path/$asm.partition.ovljob") or caExit("can't open '$path/$asm.partition.ovljob' for reading: $!", undef); open(OPT, "< $path/$asm.partition.ovlopt") or caExit("can't open '$path/$asm.partition.ovlopt' for reading: $!", undef); my @bat = ; chomp @bat; my @job = ; chomp @job; my @opt = ; chomp @opt; close(BAT); close(JOB); close(OPT); #getAllowedResources($tag, "ovl"); if (! -e "$path/overlap.sh") { my $merSize = getGlobal("${tag}OvlMerSize"); #my $hashLibrary = getGlobal("${tag}OvlHashLibrary"); #my $refLibrary = getGlobal("${tag}OvlRefLibrary"); # Create a script to run overlaps. We make a giant job array for this -- we need to know # hashBeg, hashEnd, refBeg and refEnd -- from that we compute batchName and jobName. my $hashBits = getGlobal("${tag}OvlHashBits"); my $hashLoad = getGlobal("${tag}OvlHashLoad"); open(F, "> $path/overlap.sh") or caExit("can't open '$path/overlap.sh' for writing: $!", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "perl='/usr/bin/env perl'\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; for (my $ii=1; $ii<=scalar(@bat); $ii++) { print F "if [ \$jobid -eq $ii ] ; then\n"; print F " bat=\"$bat[$ii-1]\"\n"; # Needed to mkdir. print F " job=\"$bat[$ii-1]/$job[$ii-1]\"\n"; # Needed to simplify overlapCheck() below. print F " opt=\"$opt[$ii-1]\"\n"; print F "fi\n"; print F "\n"; } print F "\n"; print F "if [ ! -d $path/\$bat ]; then\n"; print F " mkdir $path/\$bat\n"; print F "fi\n"; print F "\n"; print F "if [ -e $path/\$job.ovb.gz ]; then\n"; print F " echo Job previously completed successfully.\n"; print F " exit\n"; print F "fi\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; print F "\$bin/overlapInCore \\\n"; print F " -G \\\n" if ($type eq "partial"); print F " -t ", getGlobal("${tag}OvlThreads"), " \\\n"; print F " -k $merSize \\\n"; print F " -k $wrk/0-mercounts/$asm.ms$merSize.frequentMers.fasta \\\n"; print F " --hashbits $hashBits \\\n"; print F " --hashload $hashLoad \\\n"; print F " --maxerate ", getGlobal("${tag}OvlErrorRate"), " \\\n"; print F " --minlength ", getGlobal("minOverlapLength"), " \\\n"; print F " \$opt \\\n"; print F " -o $path/\$job.ovb.WORKING.gz \\\n"; print F " -s $path/\$job.stats \\\n"; #print F " -H $hashLibrary \\\n" if ($hashLibrary ne "0"); #print F " -R $refLibrary \\\n" if ($refLibrary ne "0"); print F " $wrk/$asm.gkpStore \\\n"; print F "&& \\\n"; print F "mv $path/\$job.ovb.WORKING.gz $path/\$job.ovb.gz\n"; print F "\n"; print F "exit 0\n"; close(F); system("chmod +x $path/overlap.sh"); } my $jobs = scalar(@job); my $batchName = $bat[$jobs-1]; chomp $batchName; my $jobName = $job[$jobs-1]; chomp $jobName; if (-e "$path/overlap.sh") { my $numJobs = 0; open(F, "< $path/overlap.sh") or caExit("can't open '$path/overlap.sh' for reading: $!", undef); while () { $numJobs++ if (m/^\s+job=/); } close(F); print STDERR "--\n"; print STDERR "-- Configured $numJobs overlapInCore jobs.\n"; } finishStage: emitStage($WRK, $asm, "$tag-overlapConfigure"); buildHTML($WRK, $asm, $tag); stopAfter("overlapConfigure"); allDone: } sub reportSumMeanStdDev (@) { my $sum = 0; my $mean = 0; my $stddev = 0; my $n = scalar(@_); my $formatted; $sum += $_ foreach (@_); $mean = $sum / $n; $stddev += ($_ - $mean) * ($_ - $mean) foreach (@_); $stddev = int(1000 * sqrt($stddev / ($n-1))) / 1000 if ($n > 1); $formatted = substr(" $mean", -10) . " +- $stddev"; return($sum, $formatted); } sub reportOverlapStats ($$@) { my $wrk = shift @_; # Local work directory my $asm = shift @_; my @statsJobs = @_; my @hitsWithoutOlaps; my @hitsWithOlaps; my @multiOlaps; my @totalOlaps; my @containedOlaps; my @dovetailOlaps; my @shortReject; my @longReject; foreach my $s (@statsJobs) { open(F, "< $s") or caExit("can't open '$s' for reading: $!", undef); $_ = ; push @hitsWithoutOlaps, $1 if (m/^\s*Kmer\shits\swithout\solaps\s=\s(\d+)$/); $_ = ; push @hitsWithOlaps, $1 if (m/^\s*Kmer\shits\swith\solaps\s=\s(\d+)$/); $_ = ; push @multiOlaps, $1 if (m/^\s*Multiple\soverlaps\/pair\s=\s(\d+)$/); $_ = ; push @totalOlaps, $1 if (m/^\s*Total\soverlaps\sproduced\s=\s(\d+)$/); $_ = ; push @containedOlaps, $1 if (m/^\s*Contained\soverlaps\s=\s(\d+)$/); $_ = ; push @dovetailOlaps, $1 if (m/^\s*Dovetail\soverlaps\s=\s(\d+)$/); $_ = ; push @shortReject, $1 if (m/^\s*Rejected\sby\sshort\swindow\s=\s(\d+)$/); $_ = ; push @longReject, $1 if (m/^\s*Rejected\sby\slong\swindow\s=\s(\d+)$/); close(F); } printf STDERR "--\n"; printf STDERR "-- overlapInCore compute '$wrk/1-overlapper':\n"; printf STDERR "-- kmer hits\n"; printf STDERR "-- with no overlap %12d %s\n", reportSumMeanStdDev(@hitsWithoutOlaps); printf STDERR "-- with an overlap %12d %s\n", reportSumMeanStdDev(@hitsWithOlaps); printf STDERR "--\n"; printf STDERR "-- overlaps %12d %s\n", reportSumMeanStdDev(@totalOlaps); printf STDERR "-- contained %12d %s\n", reportSumMeanStdDev(@containedOlaps); printf STDERR "-- dovetail %12d %s\n", reportSumMeanStdDev(@dovetailOlaps); printf STDERR "--\n"; printf STDERR "-- overlaps rejected\n"; printf STDERR "-- multiple per pair %12d %s\n", reportSumMeanStdDev(@multiOlaps); printf STDERR "-- bad short window %12d %s\n", reportSumMeanStdDev(@shortReject); printf STDERR "-- bad long window %12d %s\n", reportSumMeanStdDev(@longReject); } # Check that the overlapper jobs properly executed. If not, # complain, but don't help the user fix things. # sub overlapCheck ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $type = shift @_; my $attempt = getGlobal("canuIteration"); $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); my $path = "$wrk/1-overlapper"; goto allDone if (skipStage($WRK, $asm, "$tag-overlapCheck", $attempt) == 1); goto allDone if (-e "$path/ovljob.files"); goto allDone if (-e "$wrk/$asm.ovlStore"); # Figure out if all the tasks finished correctly. my $currentJobID = 1; my @successJobs; my @statsJobs; my @miscJobs; my @failedJobs; my $failureMessage = ""; open(F, "< $path/overlap.sh") or caExit("can't open '$path/overlap.sh' for reading: $!", undef); while () { if (m/^\s+job=\"(\d+\/\d+)\"$/) { if (-e "$path/$1.ovb.gz") { push @successJobs, "$path/$1.ovb.gz\n"; # Dumped to a file, so include \n push @statsJobs, "$path/$1.stats"; # Used here, don't include \n push @miscJobs, "$path/$1.stats\n"; push @miscJobs, "$path/$1.counts\n"; } elsif (-e "$path/$1.ovb") { push @successJobs, "$path/$1.ovb\n"; push @statsJobs, "$path/$1.stats"; push @miscJobs, "$path/$1.stats\n"; push @miscJobs, "$path/$1.counts\n"; } elsif (-e "$path/$1.ovb.bz2") { push @successJobs, "$path/$1.ovb.bz2\n"; push @statsJobs, "$path/$1.stats"; push @miscJobs, "$path/$1.stats\n"; push @miscJobs, "$path/$1.counts\n"; } elsif (-e "$path/$1.ovb.xz") { push @successJobs, "$path/$1.ovb.xz\n"; push @statsJobs, "$path/$1.stats"; push @miscJobs, "$path/$1.stats\n"; push @miscJobs, "$path/$1.counts\n"; } else { $failureMessage .= "-- job $path/$1 FAILED.\n"; push @failedJobs, $currentJobID; } $currentJobID++; } } close(F); # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " overlapper jobs failed:\n"; print STDERR $failureMessage; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to overlap. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- overlapInCore attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "$tag-overlapCheck", $attempt); buildHTML($WRK, $asm, $tag); submitOrRunParallelJob($WRK, $asm, "${tag}ovl", $path, "overlap", @failedJobs); return; } finishStage: print STDERR "-- Found ", scalar(@successJobs), " overlapInCore output files.\n"; open(L, "> $path/ovljob.files") or caExit("can't open '$path/ovljob.files' for writing: $!", undef); print L @successJobs; close(L); open(L, "> $path/ovljob.more.files") or caExit("can't open '$path/ovljob.more.files' for writing: $!", undef); print L @miscJobs; close(L); reportOverlapStats($wrk, $asm, @statsJobs); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "$tag-overlapCheck"); buildHTML($WRK, $asm, $tag); stopAfter("overlapper"); allDone: } canu-1.3/src/pipelines/canu/OverlapMMap.pm000066400000000000000000000524421272335422300205430ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Sergey Koren beginning on 2016-FEB-24 # are a 'United States Government Work', and # are released in the public domain # # Brian P. Walenz beginning on 2016-MAY-02 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::OverlapMMap; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(mmapConfigure mmapPrecomputeCheck mmapCheck); use strict; use File::Path qw(make_path remove_tree); use canu::Defaults; use canu::Execution; use canu::Gatekeeper; use canu::HTML; # Map long reads to long reads with minimap. sub mmapConfigure ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $typ = shift @_; my $bin = getBinDirectory(); $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); my $path = "$wrk/1-overlapper"; caFailure("invalid type '$typ'", undef) if (($typ ne "partial") && ($typ ne "normal")); goto allDone if (skipStage($WRK, $asm, "$tag-mmapConfigure") == 1); goto allDone if (-e "$path/mmap.sh"); goto allDone if (-e "$path/ovljob.files"); goto allDone if (-e "$wrk/$asm.ovlStore"); print STDERR "--\n"; print STDERR "-- OVERLAPPER (mmap) (correction)\n" if ($tag eq "cor"); print STDERR "-- OVERLAPPER (mmap) (trimming)\n" if ($tag eq "obt"); print STDERR "-- OVERLAPPER (mmap) (assembly)\n" if ($tag eq "utg"); print STDERR "--\n"; make_path("$path") if (! -d "$path"); # Constants. my $merSize = getGlobal("${tag}MMapMerSize"); my $numReads = getNumberOfReadsInStore($wrk, $asm); my $memorySize = getGlobal("${tag}mmapMemory"); my $blockPerGb = getGlobal("${tag}MMapBlockSize"); my $blockSize = int($blockPerGb * $memorySize); print STDERR "-- Given $memorySize GB, can fit $blockSize reads per block.\n"; # Divide the reads into blocks of ovlHashBlockSize. Each one of these blocks is used as the # table in mmap. Several of these blocks are used as the queries. my @blocks; # Range of reads to extract for this block my @blockBgn; # First read in the block my @blockLen; # Number of reads in the block my @hashes; # One for each job, the block that is the hash table my @skipSelf; # One for each job, jobs that would search block N against hash block N need to be handled special my @convert; # One for each job, flags to the mmap-ovl conversion program push @blocks, "no zeroth block, makes the loop where this is used easier"; push @blockBgn, "no zeroth block"; push @blockLen, "no zeroth block"; push @hashes, "no zeroth job"; push @skipSelf, "no zeroth job"; for (my $bgn=1; $bgn < $numReads; $bgn += $blockSize) { my $end = $bgn + $blockSize - 1; $end = $numReads if ($end > $numReads); #print STDERR "BLOCK ", scalar(@blocks), " reads from $bgn through $end\n"; push @blocks, "-b $bgn -e $end"; push @blockBgn, $bgn; push @blockLen, $end - $bgn + 1; } # Each mmap job will process one block against a set of other blocks. We'll pick, arbitrarily, # to use num_blocks/4 for that size, unless it is too small. my $numBlocks = scalar(@blocks); my $qryStride = ($numBlocks < 16) ? (2) : int($numBlocks / 4); print STDERR "-- For $numBlocks blocks, set stride to $qryStride blocks.\n"; print STDERR "-- Logging partitioning to '$path/partitioning.log'.\n"; open(L, "> $path/partitioning.log") or caExit("can't open '$path/partitioning.log' for writing: $!\n", undef); # Make queries. Each hask block needs to search against all blocks less than or equal to it. # Each job will search at most $qryStride blocks at once. So, queries could be: # 1: 1 vs 1,2,3 (with self-allowed, and quert block 1 implicitly included) # 2: 1 vs 4,5 (with no-self allowed) # 3: 2 vs 2,3,4 # 4: 2 vs 5 # 5: 3 vs 3,4,5 # 6: 4 vs 4,5 # 7: 5 vs 5 make_path("$path/queries"); for (my $bid=1; $bid < $numBlocks; $bid++) { # Note that we never do qbgn = bid; the self-self overlap is special cased. for (my $qbgn = $bid; $qbgn < $numBlocks; $qbgn += $qryStride) { my $andSelf = ""; if ($bid == $qbgn) { # The hash block bid is in the range and we need to compute hash-to-hash # overlaps, and exclude the block from the queries. push @skipSelf, "true"; $qbgn++; $andSelf = " (and self)"; } else { # The hash block bid isn't in the range of query blocks, don't allow hash-to-hash # overlaps. push @skipSelf, ""; } my $qend = $qbgn + $qryStride - 1; # Block bid searches reads in dat files from $qend = $numBlocks-1 if ($qend >= $numBlocks); # qbgn to qend (inclusive). my $job = substr("000000" . scalar(@hashes), -6); # Unique ID for this compute # Make a place to save queries. If this is the last-block-special-case, make a directory, # but don't link in any files. Without the directory, we'd need even more special case # code down in mmap.sh to exclude the -q option for this last block. make_path("$path/queries/$job"); if ($qbgn < $numBlocks) { print L "Job ", scalar(@hashes), " computes block $bid vs blocks $qbgn-$qend$andSelf,\n"; for (my $qid=$qbgn; $qid <= $qend; $qid++) { my $qry = substr("000000" . $qid, -6); # Name for the query block symlink("$path/blocks/$qry.fasta", "$path/queries/$job/$qry.fasta"); } } else { print L "Job ", scalar(@hashes), " computes block $bid vs itself.\n"; $qbgn = $bid; # Otherwise, the @convert -q value is bogus } # This is easy, the ID of the hash. push @hashes, substr("000000" . $bid, -6); # One new job for block bid with qend-qbgn query files in it } } close(L); # Create a script to generate precomputed blocks, including extracting the reads from gkpStore. open(F, "> $path/precompute.sh") or caFailure("can't open '$path/precompute.sh' for writing: $!", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; for (my $ii=1; $ii < scalar(@blocks); $ii++) { print F "if [ \$jobid -eq $ii ] ; then\n"; print F " rge=\"$blocks[$ii]\"\n"; print F " job=\"", substr("000000" . $ii, -6), "\"\n"; print F "fi\n"; print F "\n"; } print F "\n"; print F "if [ x\$job = x ] ; then\n"; print F " echo Job partitioning error. jobid \$jobid is invalid.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "if [ ! -d $path/blocks ]; then\n"; print F " mkdir $path/blocks\n"; print F "fi\n"; print F "\n"; print F "if [ -e $path/blocks/\$job.fasta ]; then\n"; print F " echo Job previously completed successfully.\n"; print F " exit\n"; print F "fi\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; print F "\$bin/gatekeeperDumpFASTQ \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " \$rge \\\n"; print F " -nolibname \\\n"; print F " -fasta \\\n"; print F " -o $path/blocks/\$job \\\n"; print F "|| \\\n"; print F "mv -f $path/blocks/\$job.fasta $path/blocks/\$job.fasta.FAILED\n"; print F "\n"; print F "\n"; print F "exit 0\n"; close(F); # Create a script to run mmap. open(F, "> $path/mmap.sh") or caFailure("can't open '$path/mmap.sh' for writing: $!", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; for (my $ii=1; $ii < scalar(@hashes); $ii++) { print F "if [ \$jobid -eq $ii ] ; then\n"; print F " blk=\"$hashes[$ii]\"\n"; print F " slf=\"$skipSelf[$ii]\"\n"; print F " qry=\"", substr("000000" . $ii, -6), "\"\n"; print F "fi\n"; print F "\n"; } print F "\n"; print F "if [ x\$qry = x ]; then\n"; print F " echo Error: Job index out of range.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "if [ -e $path/results/\$qry.ovb.gz ]; then\n"; print F " echo Job previously completed successfully.\n"; print F " exit\n"; print F "fi\n"; print F "\n"; print F "if [ ! -d $path/results ]; then\n"; print F " mkdir $path/results\n"; print F "fi\n"; print F "\n"; print F "echo Running block \$blk in query \$qry\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; # begin comparison, we loop through query and compare current block to it, if we need to do self first compare to self, otherwise initialize as empty print F "if [ x\$slf = x ]; then\n"; print F " > $path/results/\$qry.mmap.WORKING\n"; print F "else\n"; print F " \$bin/minimap \\\n"; print F " -k $merSize \\\n"; print F " -Sw5 \\\n"; print F " -L100 \\\n"; print F " -m0 \\\n"; print F " -t ", getGlobal("${tag}mmapThreads"), " \\\n"; print F " $path/blocks/\$blk.fasta \\\n"; print F " $path/blocks/\$blk.fasta \\\n"; print F " > $path/results/\$qry.mmap.WORKING \n"; print F " \n"; print F "fi\n"; print F "\n"; print F "for file in `ls $path/queries/\$qry/*.fasta`; do\n"; print F " \$bin/minimap \\\n"; print F " -k $merSize \\\n"; print F " -Sw5 \\\n"; print F " -L100 \\\n"; print F " -m0 \\\n"; print F " -t ", getGlobal("${tag}mmapThreads"), " \\\n"; print F " $path/blocks/\$blk.fasta \\\n"; print F " \$file \\\n"; print F " >> $path/results/\$qry.mmap.WORKING \n"; print F "done\n"; print F "\n"; print F "mv $path/results/\$qry.mmap.WORKING $path/results/\$qry.mmap\n"; print F "\n"; print F "if [ -e \"$path/results/\$qry.mmap\" -a \\\n"; print F " ! -e \"$path/results/\$qry.ovb.gz\" ] ; then\n"; print F " \$bin/mmapConvert \\\n"; print F " -o $path/results/\$qry.mmap.ovb.WORKING.gz \\\n"; print F " $path/results/\$qry.mmap \\\n"; print F " && \\\n"; print F " mv $path/results/\$qry.mmap.ovb.WORKING.gz $path/results/\$qry.mmap.ovb.gz\n"; print F "fi\n"; print F "\n"; if (getGlobal('saveOverlaps') eq "0") { print F "if [ -e \"$path/results/\$qry.mmap\" -a \\\n"; print F " -e \"$path/results/\$qry.mmap.ovb.gz\" ] ; then\n"; print F " rm -f $path/results/\$qry.mmap\n"; print F "fi\n"; print F "\n"; } print F "if [ -e \"$path/results/\$qry.mmap.ovb.gz\" ] ; then\n"; if (getGlobal("${tag}ReAlign") eq "raw") { print F " \$bin/overlapPair \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -O $path/results/\$qry.mmap.ovb.gz \\\n"; print F " -o $path/results/\$qry.ovb.gz \\\n"; print F " -partial \\\n" if ($typ eq "partial"); print F " -erate ", getGlobal("corErrorRate"), " \\\n" if ($tag eq "cor"); print F " -erate ", getGlobal("obtOvlErrorRate"), " \\\n" if ($tag eq "obt"); print F " -erate ", getGlobal("utgOvlErrorRate"), " \\\n" if ($tag eq "utg"); print F " -memory " . getGlobal("${tag}mmapMemory") . " \\\n"; print F " -t " . getGlobal("${tag}mmapThreads") . " \n"; } else { print F " mv -f \"$path/results/\$qry.mmap.ovb.gz\" \"$path/results/\$qry.ovb.gz\"\n"; } print F "fi\n"; print F "\n"; print F "\n"; #print F "rm -rf $path/queries/\$qry\n"; print F "\n"; print F "exit 0\n"; close(F); if (-e "$path/precompute.sh") { my $numJobs = 0; open(F, "< $path/precompute.sh") or caFailure("can't open '$path/precompute.sh' for reading: $!", undef); while () { $numJobs++ if (m/^\s+job=/); } close(F); print STDERR "-- Configured $numJobs mmap precompute jobs.\n"; } if (-e "$path/mmap.sh") { my $numJobs = 0; open(F, "< $path/mmap.sh") or caFailure("can't open '$path/mmap.sh' for reading: $!", undef); while () { $numJobs++ if (m/^\s+qry=\"(\d+)\"$/); } close(F); print STDERR "-- Configured $numJobs mmap overlap jobs.\n"; } finishStage: emitStage($WRK, $asm, "$tag-mmapConfigure"); buildHTML($WRK, $asm, $tag); stopAfter("mmapConfigure"); allDone: } sub mmapPrecomputeCheck ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $typ = shift @_; my $attempt = getGlobal("canuIteration"); $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); my $path = "$wrk/1-overlapper"; goto allDone if (skipStage($WRK, $asm, "$tag-mmapPrecomputeCheck", $attempt) == 1); goto allDone if (-e "$path/precompute.files"); goto allDone if (-e "$wrk/$asm.ovlStore"); # Figure out if all the tasks finished correctly. my $currentJobID = 1; my @successJobs; my @failedJobs; my $failureMessage = ""; open(F, "< $path/precompute.sh") or caFailure("can't open '$path/precompute.sh' for reading: $!", undef); while () { if (m/^\s+job=\"(\d+)\"$/) { if (-e "$path/blocks/$1.fasta") { push @successJobs, "$path/blocks/$1.fasta\n"; } else { $failureMessage .= "-- job $path/blocks/$1.fasta FAILED.\n"; push @failedJobs, $currentJobID; } $currentJobID++; } } close(F); # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " mmap precompute jobs failed:\n"; print STDERR $failureMessage; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to precompute mmap indices. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- mmap precompute attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "$tag-mmapPrecomputeCheck", $attempt); buildHTML($WRK, $asm, $tag); submitOrRunParallelJob($WRK, $asm, "${tag}mmap", $path, "precompute", @failedJobs); return; } finishStage: print STDERR "-- All ", scalar(@successJobs), " mmap precompute jobs finished successfully.\n"; open(L, "> $path/precompute.files") or caExit("failed to open '$path/precompute.files'", undef); print L @successJobs; close(L); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "$tag-mmapPrecomputeCheck"); buildHTML($WRK, $asm, $tag); stopAfter("mmapPrecompute"); allDone: } sub mmapCheck ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $typ = shift @_; my $attempt = getGlobal("canuIteration"); $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); my $path = "$wrk/1-overlapper"; goto allDone if (skipStage($WRK, $asm, "$tag-mmapCheck", $attempt) == 1); goto allDone if (-e "$path/mmap.files"); goto allDone if (-e "$wrk/$asm.ovlStore"); # Figure out if all the tasks finished correctly. my $currentJobID = 1; my @mmapJobs; my @successJobs; my @miscJobs; my @failedJobs; my $failureMessage = ""; open(F, "< $path/mmap.sh") or caExit("failed to open '$path/mmap.sh'", undef); while () { if (m/^\s+qry=\"(\d+)\"$/) { if (-e "$path/results/$1.ovb.gz") { push @mmapJobs, "$path/results/$1.mmap\n"; push @successJobs, "$path/results/$1.ovb.gz\n"; push @miscJobs, "$path/results/$1.stats\n"; push @miscJobs, "$path/results/$1.counts\n"; } elsif (-e "$path/results/$1.ovb") { push @mmapJobs, "$path/results/$1.mmap\n"; push @successJobs, "$path/results/$1.ovb\n"; push @miscJobs, "$path/results/$1.stats\n"; push @miscJobs, "$path/results/$1.counts\n"; } elsif (-e "$path/results/$1.ovb.bz2") { push @mmapJobs, "$path/results/$1.mmap\n"; push @successJobs, "$path/results/$1.ovb.bz2\n"; push @miscJobs, "$path/results/$1.stats\n"; push @miscJobs, "$path/results/$1.counts\n"; } elsif (-e "$path/results/$1.ovb.xz") { push @mmapJobs, "$path/results/$1.mmap\n"; push @successJobs, "$path/results/$1.ovb.xz\n"; push @miscJobs, "$path/results/$1.stats\n"; push @miscJobs, "$path/results/$1.counts\n"; } else { $failureMessage .= "-- job $path/results/$1.ovb FAILED.\n"; push @failedJobs, $currentJobID; } $currentJobID++; } } close(F); # Also find the queries symlinks so we can remove those. And the query directories, because # the last directory can be empty, and so we'd never see it at all if only finding files. open(F, "find $path/queries -print |"); while () { push @mmapJobs, $_; } close(F); # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " mmap jobs failed:\n"; print STDERR $failureMessage; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to compute mmap overlaps. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- mmap attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "$tag-mmapCheck", $attempt); buildHTML($WRK, $asm, $tag); submitOrRunParallelJob($WRK, $asm, "${tag}mmap", $path, "mmap", @failedJobs); return; } finishStage: print STDERR "-- Found ", scalar(@successJobs), " mmap overlap output files.\n"; open(L, "> $path/mmap.files") or caExit("failed to open '$path/mmap.files'", undef); print L @mmapJobs; close(L); open(L, "> $path/ovljob.files") or caExit("failed to open '$path/ovljob.files'", undef); print L @successJobs; close(L); open(L, "> $path/ovljob.more.files") or caExit("failed to open '$path/ovljob.more.files'", undef); print L @miscJobs; close(L); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "$tag-mmapCheck"); buildHTML($WRK, $asm, $tag); stopAfter("mmapOverlap"); allDone: } canu-1.3/src/pipelines/canu/OverlapMhap.pm000066400000000000000000000704531272335422300206000ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/OverlapMhap.pm # # Modifications by: # # Brian P. Walenz from 2015-MAR-27 to 2015-SEP-21 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-NOV-03 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-20 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::OverlapMhap; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(mhapConfigure mhapPrecomputeCheck mhapCheck); use strict; use POSIX qw(floor); use File::Path qw(make_path remove_tree); use canu::Defaults; use canu::Execution; use canu::Gatekeeper; use canu::HTML; # Map long reads to long reads with mhap. # Problems: # - mhap .dat output can't be verified. if the job is killed, a partial .dat is output. It needs to write to # .dat.WORKING, then let the script rename it when the job finishes successfully. There is no output file name option. sub mhapConfigure ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $typ = shift @_; my $bin = getBinDirectory(); $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); my $path = "$wrk/1-overlapper"; caFailure("invalid type '$typ'", undef) if (($typ ne "partial") && ($typ ne "normal")); goto allDone if (skipStage($WRK, $asm, "$tag-mhapConfigure") == 1); goto allDone if (-e "$path/precompute.sh") && (-e "$path/mhap.sh"); goto allDone if (-e "$path/ovljob.files"); goto allDone if (-e "$wrk/$asm.ovlStore"); print STDERR "--\n"; print STDERR "-- OVERLAPPER (mhap) (correction)\n" if ($tag eq "cor"); print STDERR "-- OVERLAPPER (mhap) (trimming)\n" if ($tag eq "obt"); print STDERR "-- OVERLAPPER (mhap) (assembly)\n" if ($tag eq "utg"); print STDERR "--\n"; make_path("$path") if (! -d "$path"); # Mhap parameters - filterThreshold needs to be a string, else it is printed as 5e-06. my ($numHashes, $minNumMatches, $threshold, $ordSketch, $ordSketchMer); if (!defined(getGlobal("${tag}MhapSensitivity"))) { my $cov = getExpectedCoverage($wrk, $asm); setGlobal("${tag}MhapSensitivity", "low"); # Yup, super inefficient. The code is setGlobal("${tag}MhapSensitivity", "normal") if ($cov < 60); # compact and clear and runs once. setGlobal("${tag}MhapSensitivity", "high") if ($cov <= 30); # Live with it. print STDERR "-- Set ${tag}MhapSensitivity=", getGlobal("${tag}MhapSensitivity"), " based on read coverage of $cov.\n"; } if (getGlobal("${tag}MhapSensitivity") eq "low") { $numHashes = 256; $minNumMatches = 3; $threshold = 0.80; $ordSketch = 1000; $ordSketchMer = getGlobal("${tag}MhapOrderedMerSize") + 2; } elsif (getGlobal("${tag}MhapSensitivity") eq "normal") { $numHashes = 512; $minNumMatches = 3; $threshold = 0.78; $ordSketch = 1536; $ordSketchMer = getGlobal("${tag}MhapOrderedMerSize"); } elsif (getGlobal("${tag}MhapSensitivity") eq "high") { $numHashes = 768; $minNumMatches = 2; $threshold = 0.73; $ordSketch = 1536; $ordSketchMer = getGlobal("${tag}MhapOrderedMerSize"); } else { caFailure("invalid ${tag}MhapSensitivity=" . getGlobal("${tag}MhapSensitivity"), undef); } my $filterThreshold = getGlobal("${tag}MhapFilterThreshold"); # Constants. my $merSize = getGlobal("${tag}MhapMerSize"); my $numReads = getNumberOfReadsInStore($wrk, $asm); my $memorySize = getGlobal("${tag}mhapMemory"); my $blockPerGb = getGlobal("${tag}MhapBlockSize"); if ($numHashes >= 768) { $blockPerGb = int($blockPerGb / 2); } # quick guess parameter adjustment for corrected reads, hack for now and should better take error rate into account if (($tag eq "obt") || ($tag eq "utg")) { $numHashes /= 4; $minNumMatches = floor(1.5 * $minNumMatches); $ordSketch = floor($ordSketch / 2); $threshold = 1-getGlobal("${tag}OvlErrorRate"); $blockPerGb *= 2; } print STDERR "--\n"; print STDERR "-- PARAMETERS: hashes=$numHashes, minMatches=$minNumMatches, threshold=$threshold\n"; print STDERR "--\n"; my $blockSize = int($blockPerGb * $memorySize); print STDERR "-- Given $memorySize GB, can fit $blockSize reads per block.\n"; # Divide the reads into blocks of ovlHashBlockSize. Each one of these blocks is used as the # table in mhap. Several of these blocks are used as the queries. my @blocks; # Range of reads to extract for this block my @blockBgn; # First read in the block my @blockLen; # Number of reads in the block my @hashes; # One for each job, the block that is the hash table my @skipSelf; # One for each job, jobs that would search block N against hash block N need to be handled special my @convert; # One for each job, flags to the mhap-ovl conversion program push @blocks, "no zeroth block, makes the loop where this is used easier"; push @blockBgn, "no zeroth block"; push @blockLen, "no zeroth block"; push @hashes, "no zeroth job"; push @skipSelf, "no zeroth job"; push @convert, "no zeroth job"; for (my $bgn=1; $bgn < $numReads; $bgn += $blockSize) { my $end = $bgn + $blockSize - 1; $end = $numReads if ($end > $numReads); #print STDERR "BLOCK ", scalar(@blocks), " reads from $bgn through $end\n"; push @blocks, "-b $bgn -e $end"; push @blockBgn, $bgn; push @blockLen, $end - $bgn + 1; } # Each mhap job will process one block against a set of other blocks. We'll pick, arbitrarily, # to use num_blocks/4 for that size, unless it is too small. my $numBlocks = scalar(@blocks); my $qryStride = ($numBlocks < 16) ? (2) : int($numBlocks / 4); print STDERR "-- For $numBlocks blocks, set stride to $qryStride blocks.\n"; print STDERR "-- Logging partitioning to '$path/partitioning.log'.\n"; open(L, "> $path/partitioning.log") or caExit("can't open '$path/partitioning.log' for writing: $!\n", undef); # Make queries. Each hask block needs to search against all blocks less than or equal to it. # Each job will search at most $qryStride blocks at once. So, queries could be: # 1: 1 vs 1,2,3 (with self-allowed, and quert block 1 implicitly included) # 2: 1 vs 4,5 (with no-self allowed) # 3: 2 vs 2,3,4 # 4: 2 vs 5 # 5: 3 vs 3,4,5 # 6: 4 vs 4,5 # 7: 5 vs 5 make_path("$path/queries"); for (my $bid=1; $bid < $numBlocks; $bid++) { # Note that we never do qbgn = bid; the self-self overlap is special cased. for (my $qbgn = $bid; $qbgn < $numBlocks; $qbgn += $qryStride) { # mhap's read labeling is dumb. It dumps all reads into one array, labeling the hash # reads 0 to N, and the query reads N+1 to N+1+M. This makes it impossible to tell if # we are comparing a read against itself (e.g., when comparing hash block 1 against # query blocks 1,2,3,4 -- compared to hash block 1 against query blocks 5,6,7,8). # # So, there is a giant special case to compare the hash block against itself enabled by # default. This needs to be disabled for the second example above. # # This makes for ANOTHER special case, that of the last block. There are no query # blocks, just the hash block, and we need to omit the flag...do we? my $andSelf = ""; if ($bid == $qbgn) { # The hash block bid is in the range and we need to compute hash-to-hash # overlaps, and exclude the block from the queries. push @skipSelf, ""; $qbgn++; $andSelf = " (and self)"; } else { # The hash block bid isn't in the range of query blocks, don't allow hash-to-hash # overlaps. push @skipSelf, "--no-self"; } my $qend = $qbgn + $qryStride - 1; # Block bid searches reads in dat files from $qend = $numBlocks-1 if ($qend >= $numBlocks); # qbgn to qend (inclusive). my $job = substr("000000" . scalar(@hashes), -6); # Unique ID for this compute # Make a place to save queries. If this is the last-block-special-case, make a directory, # but don't link in any files. Without the directory, we'd need even more special case # code down in mhap.sh to exclude the -q option for this last block. make_path("$path/queries/$job"); if ($qbgn < $numBlocks) { print L "Job ", scalar(@hashes), " computes block $bid vs blocks $qbgn-$qend$andSelf,\n"; for (my $qid=$qbgn; $qid <= $qend; $qid++) { my $qry = substr("000000" . $qid, -6); # Name for the query block symlink("$path/blocks/$qry.dat", "$path/queries/$job/$qry.dat"); } } else { print L "Job ", scalar(@hashes), " computes block $bid vs itself.\n"; $qbgn = $bid; # Otherwise, the @convert -q value is bogus } # This is easy, the ID of the hash. push @hashes, substr("000000" . $bid, -6); # One new job for block bid with qend-qbgn query files in it # Annoyingly, if we're against 'self', then the conversion needs to know that the query IDs # aren't offset by the number of hash reads. if ($andSelf eq "") { push @convert, "-h $blockBgn[$bid] $blockLen[$bid] -q $blockBgn[$qbgn]"; } else { push @convert, "-h $blockBgn[$bid] 0 -q $blockBgn[$bid]"; } } } close(L); # The ignore file is created in Meryl.pm # Create a script to generate precomputed blocks, including extracting the reads from gkpStore. #getAllowedResources($tag, "mhap"); my $javaPath = getGlobal("java"); open(F, "> $path/precompute.sh") or caFailure("can't open '$path/precompute.sh' for writing: $!", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; for (my $ii=1; $ii < scalar(@blocks); $ii++) { print F "if [ \$jobid -eq $ii ] ; then\n"; print F " rge=\"$blocks[$ii]\"\n"; print F " job=\"", substr("000000" . $ii, -6), "\"\n"; print F "fi\n"; print F "\n"; } print F "\n"; print F "if [ x\$job = x ] ; then\n"; print F " echo Job partitioning error. jobid \$jobid is invalid.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "if [ ! -d $path/blocks ]; then\n"; print F " mkdir $path/blocks\n"; print F "fi\n"; print F "\n"; print F "if [ -e $path/blocks/\$job.dat ]; then\n"; print F " echo Job previously completed successfully.\n"; print F " exit\n"; print F "fi\n"; print F "\n"; print F "# If the fasta exists, our job failed, and we should try again.\n"; print F "if [ -e \"$path/blocks/\$job.fasta\" ] ; then\n"; print F " rm -f $path/blocks/\$job.dat\n"; print F "fi\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; print F "\$bin/gatekeeperDumpFASTQ \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " \$rge \\\n"; print F " -nolibname \\\n"; print F " -fasta \\\n"; print F " -o $path/blocks/\$job \\\n"; print F "|| \\\n"; print F "mv -f $path/blocks/\$job.fasta $path/blocks/\$job.fasta.FAILED\n"; print F "\n"; print F "\n"; print F "if [ ! -e \"$path/blocks/\$job.fasta\" ] ; then\n"; print F " echo Failed to extract fasta.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "echo Starting mhap precompute.\n"; print F "\n"; print F "# So mhap writes its output in the correct spot.\n"; print F "cd $path/blocks\n"; print F "\n"; print F "$javaPath -d64 -server -Xmx", getGlobal("${tag}mhapMemory"), "g \\\n"; print F " -jar " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "\$bin/mhap-" . getGlobal("${tag}MhapVersion") . ".jar " . ($^O eq "cygwin" ? ")" : "") . "\\\n"; print F " --repeat-weight 0.9 -k $merSize \\\n"; print F " --supress-noise 2 \\\n" if (defined(getGlobal("${tag}MhapFilterUnique")) && getGlobal("${tag}MhapFilterUnique") == 1); print F " --no-tf \\\n" if (defined(getGlobal("${tag}MhapNoTf")) && getGlobal("${tag}MhapNoTf") == 1); print F " --num-hashes $numHashes \\\n"; print F " --num-min-matches $minNumMatches \\\n"; print F " --ordered-sketch-size $ordSketch \\\n"; print F " --ordered-kmer-size $ordSketchMer \\\n"; print F " --threshold $threshold \\\n"; print F " --filter-threshold $filterThreshold \\\n"; print F " --num-threads ", getGlobal("${tag}mhapThreads"), " \\\n"; print F " -f " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz" . ($^O eq "cygwin" ? ") " : "") . "\\\n" if (-e "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz"); print F " -p " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$path/blocks/\$job.fasta" . ($^O eq "cygwin" ? ") " : "") . "\\\n"; print F " -q " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$path/blocks" .($^O eq "cygwin" ? ") " : "") . "\\\n"; print F "|| \\\n"; print F "mv -f $path/blocks/\$job.dat $path/blocks/\$job.dat.FAILED\n"; print F "\n"; print F "if [ ! -e \"$path/blocks/\$job.dat\" ] ; then\n"; print F " echo Mhap failed.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "# Clean up, remove the fasta input\n"; print F "rm -f $path/blocks/\$job.fasta\n"; print F "\n"; print F "exit 0\n"; close(F); # Create a script to run mhap. open(F, "> $path/mhap.sh") or caFailure("can't open '$path/mhap.sh' for writing: $!", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; for (my $ii=1; $ii < scalar(@hashes); $ii++) { print F "if [ \$jobid -eq $ii ] ; then\n"; print F " blk=\"$hashes[$ii]\"\n"; print F " slf=\"$skipSelf[$ii]\"\n"; print F " cvt=\"$convert[$ii]\"\n"; print F " qry=\"", substr("000000" . $ii, -6), "\"\n"; print F "fi\n"; print F "\n"; } print F "\n"; print F "if [ x\$qry = x ]; then\n"; print F " echo Error: Job index out of range.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "if [ -e $path/results/\$qry.ovb.gz ]; then\n"; print F " echo Job previously completed successfully.\n"; print F " exit\n"; print F "fi\n"; print F "\n"; print F "if [ ! -d $path/results ]; then\n"; print F " mkdir $path/results\n"; print F "fi\n"; print F "\n"; print F "echo Running block \$blk in query \$qry\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; print F "if [ ! -e \"$path/results/\$qry.mhap\" ] ; then\n"; print F " $javaPath -d64 -server -Xmx", getGlobal("${tag}mhapMemory"), "g \\\n"; print F " -jar " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "\$bin/mhap-" . getGlobal("${tag}MhapVersion") . ".jar " . ($^O eq "cygwin" ? ")" : "") . "\\\n"; print F " --repeat-weight 0.9 -k $merSize \\\n"; print F " --supress-noise 2 \\\n" if (defined(getGlobal("${tag}MhapFilterUnique")) && getGlobal("${tag}MhapFilterUnique") == 1); print F " --no-tf \\\n" if (defined(getGlobal("${tag}MhapNoTf")) && getGlobal("${tag}MhapNoTf") == 1); print F " --num-hashes $numHashes \\\n"; print F " --num-min-matches $minNumMatches \\\n"; print F " --threshold $threshold \\\n"; print F " --filter-threshold $filterThreshold \\\n"; print F " --ordered-sketch-size $ordSketch \\\n"; print F " --ordered-kmer-size $ordSketchMer \\\n"; print F " --num-threads ", getGlobal("${tag}mhapThreads"), " \\\n"; print F " -f " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz" . ($^O eq "cygwin" ? ")" : "") . "\\\n" if (-e "$wrk/0-mercounts/$asm.ms$merSize.frequentMers.ignore.gz"); print F " -s " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$path/blocks/\$blk.dat \$slf" . ($^O eq "cygwin" ? ")" : "") . "\\\n"; print F " -q " . ($^O eq "cygwin" ? "\$(cygpath -w " : "") . "$path/queries/\$qry" . ($^O eq "cygwin" ? ")" : "") . "\\\n"; print F " > $path/results/\$qry.mhap.WORKING \\\n"; print F " && \\\n"; print F " mv -f $path/results/\$qry.mhap.WORKING $path/results/\$qry.mhap\n"; print F "fi\n"; print F "\n"; print F "if [ -e \"$path/results/\$qry.mhap\" -a \\\n"; print F " ! -e \"$path/results/\$qry.ovb.gz\" ] ; then\n"; print F " \$bin/mhapConvert \\\n"; print F " \$cvt \\\n"; print F " -o $path/results/\$qry.mhap.ovb.WORKING.gz \\\n"; print F " $path/results/\$qry.mhap \\\n"; print F " && \\\n"; print F " mv $path/results/\$qry.mhap.ovb.WORKING.gz $path/results/\$qry.mhap.ovb.gz\n"; print F "fi\n"; print F "\n"; if (getGlobal('saveOverlaps') eq "0") { print F "if [ -e \"$path/results/\$qry.mhap\" -a \\\n"; print F " -e \"$path/results/\$qry.mhap.ovb.gz\" ] ; then\n"; print F " rm -f $path/results/\$qry.mhap\n"; print F "fi\n"; print F "\n"; } print F "if [ -e \"$path/results/\$qry.mhap.ovb.gz\" ] ; then\n"; if (getGlobal("${tag}ReAlign") eq "raw") { print F " \$bin/overlapPair \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -O $path/results/\$qry.mhap.ovb.gz \\\n"; print F " -o $path/results/\$qry.ovb.gz \\\n"; print F " -partial \\\n" if ($typ eq "partial"); print F " -erate ", getGlobal("corErrorRate"), " \\\n" if ($tag eq "cor"); print F " -erate ", getGlobal("obtOvlErrorRate"), " \\\n" if ($tag eq "obt"); print F " -erate ", getGlobal("utgOvlErrorRate"), " \\\n" if ($tag eq "utg"); print F " -memory " . getGlobal("${tag}mhapMemory") . " \\\n"; print F " -t " . getGlobal("${tag}mhapThreads") . " \n"; } else { print F " mv -f \"$path/results/\$qry.mhap.ovb.gz\" \"$path/results/\$qry.ovb.gz\"\n"; } print F "fi\n"; print F "\n"; print F "\n"; #print F "rm -rf $path/queries/\$qry\n"; print F "\n"; print F "exit 0\n"; close(F); if (-e "$path/precompute.sh") { my $numJobs = 0; open(F, "< $path/precompute.sh") or caFailure("can't open '$path/precompute.sh' for reading: $!", undef); while () { $numJobs++ if (m/^\s+job=/); } close(F); print STDERR "-- Configured $numJobs mhap precompute jobs.\n"; } if (-e "$path/mhap.sh") { my $numJobs = 0; open(F, "< $path/mhap.sh") or caFailure("can't open '$path/mhap.sh' for reading: $!", undef); while () { $numJobs++ if (m/^\s+qry=\"(\d+)\"$/); } close(F); print STDERR "-- Configured $numJobs mhap overlap jobs.\n"; } finishStage: emitStage($WRK, $asm, "$tag-mhapConfigure"); buildHTML($WRK, $asm, $tag); stopAfter("mhapConfigure"); allDone: } sub mhapPrecomputeCheck ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $typ = shift @_; my $attempt = getGlobal("canuIteration"); $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); my $path = "$wrk/1-overlapper"; goto allDone if (skipStage($WRK, $asm, "$tag-mhapPrecomputeCheck", $attempt) == 1); goto allDone if (-e "$path/precompute.files"); goto allDone if (-e "$wrk/$asm.ovlStore"); # Figure out if all the tasks finished correctly. my $currentJobID = 1; my @successJobs; my @failedJobs; my $failureMessage = ""; open(F, "< $path/precompute.sh") or caFailure("can't open '$path/precompute.sh' for reading: $!", undef); while () { if (m/^\s+job=\"(\d+)\"$/) { if (-e "$path/blocks/$1.dat") { push @successJobs, "$path/blocks/$1.dat\n"; } else { $failureMessage .= "-- job $path/blocks/$1.dat FAILED.\n"; push @failedJobs, $currentJobID; } $currentJobID++; } } close(F); # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " mhap precompute jobs failed:\n"; print STDERR $failureMessage; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to precompute mhap indices. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- mhap precompute attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "$tag-mhapPrecomputeCheck", $attempt); buildHTML($WRK, $asm, $tag); submitOrRunParallelJob($WRK, $asm, "${tag}mhap", $path, "precompute", @failedJobs); return; } finishStage: print STDERR "-- All ", scalar(@successJobs), " mhap precompute jobs finished successfully.\n"; open(L, "> $path/precompute.files") or caExit("failed to open '$path/precompute.files'", undef); print L @successJobs; close(L); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "$tag-mhapPrecomputeCheck"); buildHTML($WRK, $asm, $tag); stopAfter("mhapPrecompute"); allDone: } sub mhapCheck ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $typ = shift @_; my $attempt = getGlobal("canuIteration"); $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); my $path = "$wrk/1-overlapper"; goto allDone if (skipStage($WRK, $asm, "$tag-mhapCheck", $attempt) == 1); goto allDone if (-e "$path/mhap.files"); goto allDone if (-e "$wrk/$asm.ovlStore"); # Figure out if all the tasks finished correctly. my $currentJobID = 1; my @mhapJobs; my @successJobs; my @miscJobs; my @failedJobs; my $failureMessage = ""; open(F, "< $path/mhap.sh") or caExit("failed to open '$path/mhap.sh'", undef); while () { if (m/^\s+qry=\"(\d+)\"$/) { if (-e "$path/results/$1.ovb.gz") { push @mhapJobs, "$path/results/$1.mhap\n"; push @successJobs, "$path/results/$1.ovb.gz\n"; push @miscJobs, "$path/results/$1.stats\n"; push @miscJobs, "$path/results/$1.counts\n"; } elsif (-e "$path/results/$1.ovb") { push @mhapJobs, "$path/results/$1.mhap\n"; push @successJobs, "$path/results/$1.ovb\n"; push @miscJobs, "$path/results/$1.stats\n"; push @miscJobs, "$path/results/$1.counts\n"; } elsif (-e "$path/results/$1.ovb.bz2") { push @mhapJobs, "$path/results/$1.mhap\n"; push @successJobs, "$path/results/$1.ovb.bz2\n"; push @miscJobs, "$path/results/$1.stats\n"; push @miscJobs, "$path/results/$1.counts\n"; } elsif (-e "$path/results/$1.ovb.xz") { push @mhapJobs, "$path/results/$1.mhap\n"; push @successJobs, "$path/results/$1.ovb.xz\n"; push @miscJobs, "$path/results/$1.stats\n"; push @miscJobs, "$path/results/$1.counts\n"; } else { $failureMessage .= "-- job $path/results/$1.ovb FAILED.\n"; push @failedJobs, $currentJobID; } $currentJobID++; } } close(F); # Also find the queries symlinks so we can remove those. And the query directories, because # the last directory can be empty, and so we'd never see it at all if only finding files. open(F, "find $path/queries -print |"); while () { push @mhapJobs, $_; } close(F); # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " mhap jobs failed:\n"; print STDERR $failureMessage; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to compute mhap overlaps. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- mhap attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "$tag-mhapCheck", $attempt); buildHTML($WRK, $asm, $tag); submitOrRunParallelJob($WRK, $asm, "${tag}mhap", $path, "mhap", @failedJobs); return; } finishStage: print STDERR "-- Found ", scalar(@successJobs), " mhap overlap output files.\n"; open(L, "> $path/mhap.files") or caExit("failed to open '$path/mhap.files'", undef); print L @mhapJobs; close(L); open(L, "> $path/ovljob.files") or caExit("failed to open '$path/ovljob.files'", undef); print L @successJobs; close(L); open(L, "> $path/ovljob.more.files") or caExit("failed to open '$path/ovljob.more.files'", undef); print L @miscJobs; close(L); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "$tag-mhapCheck"); buildHTML($WRK, $asm, $tag); stopAfter("mhapOverlap"); allDone: } canu-1.3/src/pipelines/canu/OverlapStore.pm000066400000000000000000000602331272335422300210020ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/OverlapStore.pm # # Modifications by: # # Brian P. Walenz from 2015-FEB-27 to 2015-SEP-21 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-OCT-10 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-DEC-08 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::OverlapStore; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(createOverlapStore); use strict; use POSIX qw(ceil); use canu::Defaults; use canu::Execution; use canu::HTML; # Parallel documentation: Each overlap job is converted into a single bucket of overlaps. Within # each bucket, the overlaps are distributed into many slices, one per sort job. The sort jobs then # load the same slice from each bucket. # NOT FILTERING overlaps by error rate when building the parallel store. sub createOverlapStoreSequential ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $files = shift @_; my $bin = getBinDirectory(); my $cmd; $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); # This is running in the canu process itself. Execution.pm has special case code # to submit canu to grids using the maximum of 4gb and this memory limit. my $memSize = getGlobal("ovsMemory"); # The parallel store build will unlimit 'max user processes'. The sequential method usually # runs out of open file handles first (meaning it has never run out of processes yet). $cmd = "$bin/ovStoreBuild \\\n"; $cmd .= " -O $wrk/$asm.ovlStore.BUILDING \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -M $memSize \\\n"; $cmd .= " -L $files \\\n"; $cmd .= " > $wrk/$asm.ovlStore.err 2>&1"; if (runCommand($wrk, $cmd)) { caExit("failed to create the overlap store", "$wrk/$asm.ovlStore.err"); } unlink "$wrk/$asm.ovlStore.err"; rename "$wrk/$asm.ovlStore.BUILDING", "$wrk/$asm.ovlStore"; } # Count the number of inputs. We don't expect any to be missing (they were just checked # by overlapCheck()) but feel silly not checking again. sub countOverlapStoreInputs ($) { my $inputs = shift @_; my $numInputs = 0; open(F, "< $inputs") or die "Failed to open overlap store input file '$inputs': $0\n"; while () { chomp; die "overlapper output '$_' not found\n" if (! -e $_); $numInputs++; } close(F); return($numInputs); } sub getNumOlapsAndSlices ($$) { my $wrk = shift @_; my $asm = shift @_; my $numOlaps = undef; my $numSlices = undef; my $memLimit = undef; open(F, "< $wrk/$asm.ovlStore.BUILDING/config.err") or caExit("can't open '$wrk/$asm.ovlStore.BUILDING/config.err' for reading: $!\n", undef); while () { if (m/Will sort (\d+.\d+) million overlaps per bucket, using (\d+) buckets (\d+.\d+) GB per bucket./) { $numOlaps = $1; $numSlices = $2; $memLimit = ceil($3); } } close(F); if (!defined($numOlaps) || !defined($numSlices) || !defined($memLimit)) { caExit("Failed to find any overlaps ($numOlaps) or slices ($numSlices) or memory limit ($memLimit)", undef); } return($numOlaps, $numSlices, $memLimit); } sub overlapStoreConfigure ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $files = shift @_; my $bin = getBinDirectory(); my $cmd; $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); goto allDone if (skipStage($WRK, $asm, "$tag-overlapStoreConfigure") == 1); goto allDone if (-d "$wrk/$asm.ovlStore"); my $numInputs = countOverlapStoreInputs($files); # Create an output directory, and populate it with more directories and scripts system("mkdir -p $wrk/$asm.ovlStore.BUILDING") if (! -d "$wrk/$asm.ovlStore.BUILDING"); system("mkdir -p $wrk/$asm.ovlStore.BUILDING/scripts") if (! -d "$wrk/$asm.ovlStore.BUILDING/scripts"); system("mkdir -p $wrk/$asm.ovlStore.BUILDING/logs") if (! -d "$wrk/$asm.ovlStore.BUILDING/logs"); # Run the normal store build, but just to get the partitioning. if (! -e "$wrk/$asm.ovlStore.BUILDING/config") { $cmd = "$bin/ovStoreBuild \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -O $wrk/$asm.ovlStore \\\n"; # NOT created! $cmd .= " -M " . getGlobal("ovsMemory") . " \\\n"; $cmd .= " -config $wrk/$asm.ovlStore.BUILDING/config \\\n"; $cmd .= " -L $files \\\n"; $cmd .= "> $wrk/$asm.ovlStore.BUILDING/config.err 2>&1\n"; if (runCommand($wrk, $cmd)) { caExit("failed to generate configuration for building overlap store", "$wrk/$asm.ovlStore.BUILDING/config.err"); } } # Parse the output to find the number of jobs we need to sort and the memory # ovs store memory is left as a range (e.g. 4-16) so building can scale itself to (hopefully) fit both into memory and into max system open files my ($numOlaps, $numSlices, $memLimit) = getNumOlapsAndSlices($wrk, $asm); setGlobal("ovsMemory", ceil($memLimit * 1.1)); # request more memory to avoid memory issues on machines with no swap # Parallel jobs for bucketizing. This should really be part of overlap computation itself. #getAllowedResources("", "ovb"); if (! -e "$wrk/$asm.ovlStore.BUILDING/scripts/1-bucketize.sh") { open(F, "> $wrk/$asm.ovlStore.BUILDING/scripts/1-bucketize.sh") or die; print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "bn=`printf %04d \$jobid`\n"; print F "jn=\"undefined\"\n"; print F "\n"; my $tstid = 1; open(I, "< $files") or die "Failed to open '$files': $0\n"; while () { chomp; print F "if [ \"\$jobid\" -eq \"$tstid\" ] ; then jn=\"$_\"; fi\n"; $tstid++; } close(I); print F "\n"; print F "if [ \$jn = \"undefined\" ] ; then\n"; print F " echo \"Job out of range.\"\n"; print F " exit\n"; print F "fi\n"; print F "\n"; print F "if [ -e \"$wrk/$asm.ovlStore.BUILDING/bucket\$bn/sliceSizes\" ] ; then\n"; print F " echo \"Bucket $wrk/$asm.ovlStore.BUILDING/bucket\$bn finished successfully.\"\n"; print F " exit\n"; print F "fi\n"; print F "\n"; print F "if [ -e \"$wrk/$asm.ovlStore.BUILDING/create\$bn\" ] ; then\n"; print F " echo \"Removing incomplete bucket $wrk/$asm.ovlStore.BUILDING/create\$bn\"\n"; print F " rm -rf \"$wrk/$asm.ovlStore.BUILDING/create\$bn\"\n"; print F "fi\n"; print F "\n"; print F "max=`ulimit -Hu`\n"; print F "bef=`ulimit -Su`\n"; print F "if [ \$bef -lt \$max ] ; then\n"; print F " ulimit -Su \$max\n"; print F " aft=`ulimit -Su`\n"; print F " echo \"Changed max processes per user from \$bef to \$aft (max \$max).\"\n"; print F " echo \"\"\n"; print F "else\n"; print F " echo \"Max processes per user limited to \$bef, no increase possible.\"\n"; print F " echo \"\"\n"; print F "fi\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; print F "\$bin/ovStoreBucketizer \\\n"; print F " -O $wrk/$asm.ovlStore.BUILDING \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -C $wrk/$asm.ovlStore.BUILDING/config \\\n"; #print F " -e " . getGlobal("") . " \\\n" if (defined(getGlobal(""))); print F " -job \$jobid \\\n"; print F " -i \$jn\n"; close(F); } # Parallel jobs for sorting each bucket #getAllowedResources("", "ovs"); if (! -e "$wrk/$asm.ovlStore.BUILDING/scripts/2-sort.sh") { open(F, "> $wrk/$asm.ovlStore.BUILDING/scripts/2-sort.sh") or die; print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "jobid=\$" . getGlobal("gridEngineTaskID") . "\n"; print F "if [ x\$jobid = x -o x\$jobid = xundefined -o x\$jobid = x0 ]; then\n"; print F " jobid=\$1\n"; print F "fi\n"; print F "if [ x\$jobid = x ]; then\n"; print F " echo Error: I need " . getGlobal("gridEngineTaskID") . " set, or a job index on the command line.\n"; print F " exit 1\n"; print F "fi\n"; print F "\n"; print F "max=`ulimit -Hu`\n"; print F "bef=`ulimit -Su`\n"; print F "if [ \$bef -lt \$max ] ; then\n"; print F " ulimit -Su \$max\n"; print F " aft=`ulimit -Su`\n"; print F " echo \"Changed max processes per user from \$bef to \$aft (max \$max).\"\n"; print F " echo \"\"\n"; print F "else\n"; print F " echo \"Max processes per user limited to \$bef, no increase possible.\"\n"; print F " echo \"\"\n"; print F "fi\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; print F "\$bin/ovStoreSorter \\\n"; print F " -deletelate \\\n"; # Choices -deleteearly -deletelate or nothing print F " -M $memLimit \\\n"; print F " -O $wrk/$asm.ovlStore.BUILDING \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -F $numSlices \\\n"; print F " -job \$jobid $numInputs\n"; print F "\n"; print F "if [ \$? = 0 ] ; then\n"; print F " echo Success.\n"; print F "else\n"; print F " echo Failure.\n"; print F "fi\n"; close(F); } # A final job to merge the indices. if (! -e "$wrk/$asm.ovlStore.BUILDING/scripts/3-index.sh") { open(F, "> $wrk/$asm.ovlStore.BUILDING/scripts/3-index.sh") or die; print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; print F "\$bin/ovStoreIndexer \\\n"; #print F " -nodelete \\\n"; # Choices -nodelete or nothing print F " -O $wrk/$asm.ovlStore.BUILDING \\\n"; print F " -F $numSlices\n"; print F "\n"; print F "if [ \$? = 0 ] ; then\n"; print F " echo Success.\n"; print F "else\n"; print F " echo Failure.\n"; print F "fi\n"; close(F); } system("chmod +x $wrk/$asm.ovlStore.BUILDING/scripts/1-bucketize.sh"); system("chmod +x $wrk/$asm.ovlStore.BUILDING/scripts/2-sort.sh"); system("chmod +x $wrk/$asm.ovlStore.BUILDING/scripts/3-index.sh"); finishStage: emitStage($WRK, $asm, "$tag-overlapStoreConfigure"); buildHTML($WRK, $asm, $tag); stopAfter("overlapStoreConfigure"); allDone: } sub overlapStoreBucketizerCheck ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $files = shift @_; my $attempt = getGlobal("canuIteration"); $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); goto allDone if (skipStage($WRK, $asm, "$tag-overlapStoreBucketizerCheck", $attempt) == 1); goto allDone if (-d "$wrk/$asm.ovlStore"); goto allDone if (-e "$wrk/$asm.ovlStore.BUILDING/1-bucketize.success"); # Figure out if all the tasks finished correctly. my $numInputs = countOverlapStoreInputs($files); my $currentJobID = 1; my @successJobs; my @failedJobs; my $failureMessage = ""; my $bucketID = "0001"; # Two ways to check for completeness, either 'sliceSizes' exists, or the 'bucket' directory # exists. The compute is done in a 'create' directory, which is renamed to 'bucket' just # before the job completes. open(F, "< $files") or caExit("can't open '$files' for reading: $!", undef); while () { chomp; if (! -e "$wrk/$asm.ovlStore.BUILDING/bucket$bucketID") { $failureMessage .= "-- job $wrk/$asm.ovlStore.BUILDING/bucket$bucketID FAILED.\n"; push @failedJobs, $currentJobID; } else { push @successJobs, $currentJobID; } $currentJobID++; $bucketID++; } close(F); # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " overlap store bucketizer jobs failed:\n"; print STDERR $failureMessage; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to overlapStoreBucketize. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- overlap store bucketizer attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "$tag-overlapStoreBucketizerCheck", $attempt); buildHTML($WRK, $asm, $tag); submitOrRunParallelJob($WRK, $asm, "ovB", "$wrk/$asm.ovlStore.BUILDING", "scripts/1-bucketize", @failedJobs); return; } finishStage: print STDERR "-- Overlap store bucketizer finished.\n"; touch("$wrk/$asm.ovlStore.BUILDING/1-bucketize.success"); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "$tag-overlapStoreBucketizerCheck"); buildHTML($WRK, $asm, $tag); stopAfter("overlapBucketizer"); allDone: } sub overlapStoreSorterCheck ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $files = shift @_; my $attempt = getGlobal("canuIteration"); $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); goto allDone if (skipStage($WRK, $asm, "$tag-overlapStoreSorterCheck", $attempt) == 1); goto allDone if (-d "$wrk/$asm.ovlStore"); goto allDone if (-e "$wrk/$asm.ovlStore.BUILDING/2-sorter.success"); # Figure out if all the tasks finished correctly. my ($numOlaps, $numSlices) = getNumOlapsAndSlices($wrk, $asm); my $currentJobID = 1; my @successJobs; my @failedJobs; my $failureMessage = ""; my $sortID = "0001"; open(F, "< $files") or caExit("can't open '$files' for reading: $!", undef); # A valid result has three files: # $wrk/$asm.ovlStore.BUILDING/$sortID # $wrk/$asm.ovlStore.BUILDING/$sortID.index # $wrk/$asm.ovlStore.BUILDING/$sortID.info # # A crashed result has one file, if it crashes before output # $wrk/$asm.ovlStore.BUILDING/$sortID.ovs # # On out of disk, the .info is missing. It's the last thing created. # while ($currentJobID <= $numSlices) { if ((! -e "$wrk/$asm.ovlStore.BUILDING/$sortID") || (! -e "$wrk/$asm.ovlStore.BUILDING/$sortID.info") || ( -e "$wrk/$asm.ovlStore.BUILDING/$sortID.ovs")) { $failureMessage .= "-- job $wrk/$asm.ovlStore.BUILDING/$sortID FAILED.\n"; unlink "$wrk/$asm.ovlStore.BUILDING/$sortID.ovs"; push @failedJobs, $currentJobID; } else { push @successJobs, $currentJobID; } $currentJobID++; $sortID++; } close(F); # Failed jobs, retry. if (scalar(@failedJobs) > 0) { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- ", scalar(@failedJobs), " overlap store sorter jobs failed:\n"; print STDERR $failureMessage; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to overlapStoreSorter. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- overlap store sorter attempt $attempt begins with ", scalar(@successJobs), " finished, and ", scalar(@failedJobs), " to compute.\n"; emitStage($WRK, $asm, "$tag-overlapStoreSorterCheck", $attempt); buildHTML($WRK, $asm, $tag); submitOrRunParallelJob($WRK, $asm, "ovS", "$wrk/$asm.ovlStore.BUILDING", "scripts/2-sort", @failedJobs); return; } finishStage: print STDERR "-- Overlap store sorter finished.\n"; touch("$wrk/$asm.ovlStore.BUILDING/2-sorter.success"); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "$tag-overlapStoreSorterCheck"); buildHTML($WRK, $asm, $tag); stopAfter("overlapSorter"); allDone: } sub createOverlapStoreParallel ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $files = shift @_; $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); overlapStoreConfigure($WRK, $asm, $tag, $files); overlapStoreBucketizerCheck($WRK, $asm, $tag, $files) foreach (1..getGlobal("canuIterationMax") + 1); overlapStoreSorterCheck($WRK, $asm, $tag, $files) foreach (1..getGlobal("canuIterationMax") + 1); if (runCommand("$wrk/$asm.ovlStore.BUILDING", "$wrk/$asm.ovlStore.BUILDING/scripts/3-index.sh > $wrk/$asm.ovlStore.BUILDING/logs/3-index.err 2>&1")) { caExit("failed to build index for overlap store", "$wrk/$asm.ovlStore.BUILDING/logs/3-index.err"); } rename "$wrk/$asm.ovlStore.BUILDING", "$wrk/$asm.ovlStore"; } sub generateOverlapStoreStats ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $bin = getBinDirectory(); my $cmd; $cmd = "$bin/ovStoreStats \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -O $wrk/$asm.ovlStore \\\n"; $cmd .= " -o $wrk/$asm.ovlStore \\\n"; $cmd .= " > $wrk/$asm.ovlStore.summary.err 2>&1"; if (runCommand($wrk, $cmd)) { print STDERR "--\n"; print STDERR "-- WARNING: failed to generate statistics for the overlap store; no summary will appear in HTML output.\n"; print STDERR "--\n"; print STDERR "----------------------------------------\n"; } } sub createOverlapStore ($$$$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = $WRK; # Local work directory my $asm = shift @_; my $tag = shift @_; my $seq = shift @_; $wrk = "$wrk/correction" if ($tag eq "cor"); $wrk = "$wrk/trimming" if ($tag eq "obt"); $wrk = "$wrk/unitigging" if ($tag eq "utg"); my $path = "$wrk/1-overlapper"; goto allDone if (skipStage($WRK, $asm, "$tag-createOverlapStore") == 1); goto allDone if (-d "$wrk/$asm.ovlStore"); goto allDone if (-d "$wrk/$asm.tigStore"); # Did we _really_ complete? caExit("overlapper claims to be finished, but no job list found in '$path/ovljob.files'", undef) if (! -e "$path/ovljob.files"); # Then just build the store! Simple! createOverlapStoreSequential($WRK, $asm, $tag, "$path/ovljob.files") if ($seq eq "sequential"); createOverlapStoreParallel ($WRK, $asm, $tag, "$path/ovljob.files") if ($seq eq "parallel"); print STDERR "--\n"; print STDERR "-- Overlap store '$wrk/$asm.ovlStore' successfully constructed.\n"; goto finishStage if (getGlobal("saveOverlaps") eq "1"); # Delete the inputs and directories. Some contortions are needed to get directory deletes in order. # In particular, mhap's queries directory needs to be deleted after it's subdirectories are. my %directories; my $bytes = 0; my $files = 0; foreach my $file ("$path/ovljob.files", "$path/ovljob.more.files", "$path/mhap.files", "$path/mmap.files", "$path/precompute.files") { next if (! -e $file); open(F, "< $file") or caExit("can't open '$file' for reading: $!\n", undef); while () { chomp; # Decide what to do. Directories - register for later deletion. Files - sum size and # delete. if (-d $_) { $directories{$_}++; } elsif (-e $_) { $bytes += -s $_; $files += 1; unlink $_; } # If the file isn't a directory -- the file itself could not exist above -- register # the directory it is in for deletion. if (! -d $_) { my @components = split '/', $_; pop @components; my $dir = join '/', @components; $directories{$dir}++; } } close(F); unlink $file; } # Ideally, every directory we have in our list should be empty. But they won't be. So, loop until we fail to delete anything. my $dirs = 0; my $deleted = 1; while ($deleted > 0) { $deleted = 0; foreach my $dir (keys %directories) { if (-d $dir) { rmdir $dir; $dirs++; } if (! -d $dir) { # If really removed, remove it from our list. delete $directories{$dir}; $deleted++; } } } print STDERR "--\n"; print STDERR "-- Purged ", int(1000 * $bytes / 1024 / 1024 / 1024) / 1000, " GB in $files overlap output files and $dirs directories.\n"; # Now all done! finishStage: generateOverlapStoreStats($wrk, $asm); if (-e "$wrk/$asm.ovlStore.summary") { print STDERR "--\n"; print STDERR "-- Overlap store '$wrk/$asm.ovlStore' contains:\n"; print STDERR "--\n"; open(F, "< $wrk/$asm.ovlStore.summary") or caExit("Failed to open overlap store statistics in '$wrk/$asm.ovlStore': $!", undef); while () { print STDERR "-- $_"; } close(F); } else { print STDERR "-- Overlap store '$wrk/$asm.ovlStore' statistics not available.\n"; } emitStage($WRK, $asm, "$tag-createOverlapStore"); buildHTML($WRK, $asm, $tag); stopAfter("overlapStore"); allDone: } canu-1.3/src/pipelines/canu/Unitig.pm000066400000000000000000000226441272335422300176200ustar00rootroot00000000000000 ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # This file is derived from: # # src/pipelines/ca3g/Unitig.pm # # Modifications by: # # Brian P. Walenz from 2015-FEB-27 to 2015-AUG-26 # are Copyright 2015 Battelle National Biodefense Institute, and # are subject to the BSD 3-Clause License # # Brian P. Walenz beginning on 2015-NOV-02 # are a 'United States Government Work', and # are released in the public domain # # Sergey Koren beginning on 2015-NOV-27 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## package canu::Unitig; require Exporter; @ISA = qw(Exporter); @EXPORT = qw(reportUnitigSizes unitig unitigCheck); use strict; use File::Path qw(make_path remove_tree); use POSIX qw(ceil); use canu::Defaults; use canu::Execution; use canu::Configure; # For displayGenomeSize use canu::Gatekeeper; use canu::HTML; use canu::Meryl; sub reportUnitigSizes ($$$$) { my $wrk = shift @_; my $asm = shift @_; my $version = shift @_; my $label = shift @_; my $bin = getBinDirectory(); my $cmd = ""; my $ctgNum = 0; my $ctgBases = 0; my $ctgSizes = ""; my $usmNum = 0; my $usmBases = 0; my $bubNum = 0; my $bubBases = 0; my $rptNum = 0; my $rptBases = 0; my $gs = getGlobal("genomeSize"); my $V = substr("000000" . $version, -3); my $N = "$wrk/$asm.tigStore/seqDB.v$V.sizes.txt"; if (! -e $N) { $cmd = "$bin/tgStoreDump \\\n"; $cmd .= " -G $wrk/$asm.gkpStore \\\n"; $cmd .= " -T $wrk/$asm.tigStore $version \\\n"; $cmd .= " -sizes -s " . getGlobal("genomeSize") . " \\\n"; $cmd .= "> $N"; if (runCommand($wrk, $cmd)) { caExit("failed to generate unitig sizes", undef); } } $ctgSizes .= "-- NG (bp) LG (contigs) sum (bp)\n"; $ctgSizes .= "-- ---------- ------------ ----------\n"; open(F, "< $N") or caExit("failed to open '$N' for reading: $!\n", undef); while () { $rptBases = $1 if (m/lenSuggestRepeat\s+sum\s+(\d+)/); $rptNum = $1 if (m/lenSuggestRepeat\s+num\s+(\d+)/); $usmBases = $1 if (m/lenUnassembled\s+sum\s+(\d+)/); $usmNum = $1 if (m/lenUnassembled\s+num\s+(\d+)/); $bubBases = $1 if (m/lenBubble\s+sum\s+(\d+)/); $bubNum = $1 if (m/lenBubble\s+num\s+(\d+)/); $ctgBases = $1 if (m/lenContig\s+sum\s+(\d+)/); $ctgNum = $1 if (m/lenContig\s+num\s+(\d+)/); if (m/lenContig\s+ng(\d+)\s+(\d+)\s+bp\s+lg(\d+)\s+(\d+)\s+sum\s+(\d+)\s+bp$/) { my $n = substr(" $1", -3); my $ng = substr(" $2", -10); my $lg = substr(" $4", -6); my $ss = substr(" $5", -10); $ctgSizes .= "-- $n $ng $lg $ss\n"; } # User could have changed genome size since this report was dumped. if (m/lenContig.*genomeSize\s+(\d+)/) { $gs = $1; } } close(F); print STDERR "-- Found, in version $version, $label:\n"; print STDERR "-- contigs: $ctgNum sequences, total length $ctgBases bp (including $rptNum repeats of total length $rptBases bp).\n"; print STDERR "-- bubbles: $bubNum sequences, total length $bubBases bp.\n"; print STDERR "-- unassembled: $usmNum sequences, total length $usmBases bp.\n"; print STDERR "--\n"; print STDERR "-- Contig sizes based on genome size ", displayGenomeSize($gs), "bp:\n"; print STDERR "--\n"; print STDERR "$ctgSizes"; print STDERR "--\n"; } sub unitig ($$) { my $WRK = shift @_; # Root work directory (the -d option to canu) my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; goto allDone if (skipStage($wrk, $asm, "unitig") == 1); goto allDone if (-d "$wrk/$asm.tigStore"); make_path("$wrk/4-unitigger") if (! -d "$wrk/4-unitigger"); # How many reads per partition? This will change - it'll move to be after unitigs are constructed. my $perPart = int(getNumberOfReadsInStore($wrk, $asm) / getGlobal("cnsPartitions")); my $minPart = getGlobal("cnsPartitionMin"); my $overlapLength = getGlobal("minOverlapLength"); $perPart = ($perPart < $minPart) ? ($perPart) : ($minPart); # Dump a script to run the unitigger. open(F, "> $wrk/4-unitigger/unitigger.sh") or caExit("can't open '$wrk/4-unitigger/unitigger.sh' for writing: $!\n", undef); print F "#!" . getGlobal("shell") . "\n"; print F "\n"; print F "if [ -e $wrk/$asm.tigStore/seqDB.v001.tig ] ; then\n"; print F " exit 0\n"; print F "fi\n"; print F "\n"; print F getBinDirectoryShellCode(); print F "\n"; if (getGlobal("unitigger") eq "bogart") { print F "\$bin/bogart \\\n"; print F " -G $wrk/$asm.gkpStore \\\n"; print F " -O $wrk/$asm.ovlStore \\\n"; print F " -T $wrk/$asm.tigStore.WORKING \\\n"; print F " -o $wrk/4-unitigger/$asm \\\n"; print F " -B $perPart \\\n"; print F " -gs " . getGlobal("genomeSize") . " \\\n"; print F " -eg " . getGlobal("utgOvlErrorRate") . " \\\n"; print F " -eM " . getGlobal("utgOvlErrorRate") . " \\\n"; print F " -el " . $overlapLength . " \\\n"; print F " -dg " . getGlobal("utgGraphDeviation") . " \\\n"; print F " -db " . getGlobal("utgGraphDeviation") . " \\\n"; print F " -dr " . getGlobal("utgRepeatDeviation") . " \\\n"; print F " -ca " . getGlobal("utgRepeatConfusedBP"). " \\\n"; print F " -cp " . "500" . " \\\n"; print F " -threads " . getGlobal("batThreads") . " \\\n" if (defined(getGlobal("batThreads"))); print F " -M " . getGlobal("batMemory") . " \\\n" if (defined(getGlobal("batMemory"))); print F " -unassembled " . getGlobal("contigFilter") . " \\\n" if (defined(getGlobal("contigFilter"))); print F " " . getGlobal("batOptions") . " \\\n" if (defined(getGlobal("batOptions"))); print F " > $wrk/4-unitigger/unitigger.err 2>&1 \\\n"; print F "&& \\\n"; print F "mv $wrk/$asm.tigStore.WORKING $wrk/$asm.tigStore.FINISHED\n"; } else { caFailure("unknown unitigger '" . getGlobal("unitigger") . "'", undef); } print F "\n"; print F "exit 0\n"; close(F); finishStage: emitStage($WRK, $asm, "unitig"); buildHTML($WRK, $asm, "utg"); stopAfter("unitig"); allDone: } sub unitigCheck ($$) { my $WRK = shift @_; # Root work directory my $wrk = "$WRK/unitigging"; # Local work directory my $asm = shift @_; my $attempt = getGlobal("canuIteration"); my $path = "$wrk/4-unitigger"; goto allDone if (skipStage($WRK, $asm, "unitigCheck", $attempt) == 1); goto allDone if (-e "$wrk/$asm.tigStore/seqDB.v001.tig"); # Since there is only one job, if we get here, we're not done. Any other 'check' function # shows how to process multiple jobs. This only checks for the existence of either *.WORKING # (crashed or killed) or *.FINISHED (done). # If 'FINISHED' exists, the job finished successfully. if (! -e "$wrk/$asm.tigStore.FINISHED/seqDB.v001.tig") { # If not the first attempt, report the jobs that failed, and that we're recomputing. if ($attempt > 1) { print STDERR "--\n"; print STDERR "-- Unitigger failed.\n"; print STDERR "--\n"; } # If too many attempts, give up. if ($attempt > getGlobal("canuIterationMax")) { caExit("failed to generate unitigs. Made " . ($attempt-1) . " attempts, jobs still failed", undef); } # Otherwise, run some jobs. print STDERR "-- Unitigger attempt $attempt begins.\n"; emitStage($WRK, $asm, "unitigCheck", $attempt); buildHTML($WRK, $asm, "utg"); submitOrRunParallelJob($WRK, $asm, "bat", $path, "unitigger", (1)); return; } # If onGrid, the submitOrRun() has submitted parallel jobs to the grid, and resubmitted the # executive. # The parallel version never gets here finishStage: print STDERR "-- Unitigger finished successfully.\n"; rename "$wrk/$asm.tigStore.FINISHED", "$wrk/$asm.tigStore"; reportUnitigSizes($wrk, $asm, 1, "after unitig construction"); setGlobal("canuIteration", 0); emitStage($WRK, $asm, "unitigCheck"); buildHTML($WRK, $asm, "utg"); stopAfter("unitigCheck"); allDone: } canu-1.3/src/pipelines/install-perl-libraries.sh000077500000000000000000000047541272335422300220160ustar00rootroot00000000000000#!/bin/sh installdir=$1 # Should detect which modules we really need to install. For now, just install both. # Remove old junk rm -rf File-Path-2.09 rm -rf Filesys-Df-0.92 # Clean our environment unset CC unset CXX # Decide which perl we have. perlversion=`perl -e '$x = ($] < 5.009) ? "old" : "new"; print $x'` # Our older perl (5.8.8) uses an older MakeMaker, which doesn't know INSTALL_BASE, and installs the # Filesys::Df package in the wrong place. # # Our newer perl (5.10.1) knows INSTALL_BASE, and has File::Path installed already. if [ $perlversion = "old" ] ; then echo "Installing File-Path-2.09" echo \ tar xzf File-Path-2.09.tar.gz tar xzf File-Path-2.09.tar.gz cd File-Path-2.09 echo \ perl Makefile.PL PREFIX=$installdir perl Makefile.PL PREFIX=$installdir > make.maker.err 2>&1 echo \ make install make install > make.install.err 2>&1 cd .. echo "Installing Filesys-Df-0.92" echo \ tar xzf Filesys-Df-0.92.tar.gz tar xzf Filesys-Df-0.92.tar.gz cd Filesys-Df-0.92 echo \ perl Makefile.PL PREFIX=$installdir perl Makefile.PL PREFIX=$installdir > make.maker.err 2>&1 echo \ make install make install > make.install.err 2>&1 cd .. echo \ mv $installdir/lib64/perl5/site_perl/5*/*/Filesys $installdir/lib64/perl5/5*/*/ mv $installdir/lib64/perl5/site_perl/5*/*/Filesys $installdir/lib64/perl5/5*/*/ echo \ mv $installdir/lib64/perl5/site_perl/5*/*/auto/Filesys $installdir/lib64/perl5/5*/*/auto/ mv $installdir/lib64/perl5/site_perl/5*/*/auto/Filesys $installdir/lib64/perl5/5*/*/auto/ echo \ rm -rf $installdir/lib64/perl5/site_perl rm -rf $installdir/lib64/perl5/site_perl else echo "Installing Filesys-Df-0.92" echo \ tar xzf Filesys-Df-0.92.tar.gz tar xzf Filesys-Df-0.92.tar.gz cd Filesys-Df-0.92 echo \ perl Makefile.PL INSTALL_BASE=$installdir perl Makefile.PL INSTALL_BASE=$installdir > make.maker.err 2>& 1 # The toplevel GNU make is setting MAKEFLAGS to be "-j --jobserver-fds=3,4". The BSD make # invoked below (on FreeBSD and probably OS X) requires a value for -j. So we just remove it. # In GNU make, '-j' says to run as many tasks in parallel as possible; this isn't heavy lifting, # so won't matter if it's sequential. The jobserver-fds baloney is to track jobs in parallel, # again, we don't care. export MAKEFLAGS= echo \ make install make install > make.install.err 2>&1 cd .. fi # Remove the junk rm -rf File-Path-2.09 rm -rf Filesys-Df-0.92 canu-1.3/src/pipelines/sanity/000077500000000000000000000000001272335422300163745ustar00rootroot00000000000000canu-1.3/src/pipelines/sanity/build-all-kmer-revisions.pl000066400000000000000000000027461272335422300235620ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Given a local rsync'd copy of the repository, check out all versions and compile. my $kmersvn = "/work/NIGHTLY/kmer-svn" open(F, "< $kmersvn/db/current"); my $latest = ; chomp $latest; close(F); for (my $i=1917; $i<=$latest; $i++) { if (! -d "kmer$i") { print "Check out r$i\n"; system("mkdir kmer$i"); system("cd kmer$i && svn co -r $i file://$kmersvn/trunk . > kmer-checkout.err 2>&1"); } if (! -d "kmer$i/FreeBSD-amd64") { print "Compile r$i\n"; system("cd kmer$i && gmake install > kmer-build.err 2>& 1 &"); } } canu-1.3/src/pipelines/sanity/build-all-wgs-revisions.pl000066400000000000000000000132421272335422300234150ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## #-------------------------------------------------------------------------------- use strict; my $wgssvn = "/work/NIGHTLY/SVN-wgs"; my $kmersvn = "/work/NIGHTLY/SVN-kmer"; my $wgsbyd = "/work/NIGHTLY/bydate"; system("mkdir $wgsbyd") if (! -d "$wgsbyd"); my %revToDate; if (-d "$wgsbyd/TIP") { print STDERR "Updating to latest. DON'T FORGET TO sanity.pl rsync FIRST!\n"; system("cd $wgsbyd/TIP/src ; svn update > update.err"); } else { print STDERR "Checking out latest.\n"; system("mkdir $wgsbyd/TIP"); system("cd $wgsbyd/TIP ; svn co file://$wgssvn/trunk/src src > checkout.err"); } print STDERR "Reading CA commit logs.\n"; open(F, "cd $wgsbyd/TIP/src ; svn log |"); while () { #print STDERR $_; # CVS relic #if (m/^date:\s+(....)\/(..)\/(..)\s+(..):(..):(..);\s+author:\s+(.*);\s+state/) { # my $fildate = "$1-$2-$3-$4$5"; # my $cvsdate = "$1/$2/$3 $4:$5:$6 GMT"; # my $svndate = "{$1$2$3T$4$5Z}"; # $logdate{"$fildate\0$cvsdate\0$svndate"}++; #} if (m/^(r\d+)\s+\|\s+(.+)\s+\|\s+(\d\d\d\d)-(\d\d)-(\d\d)\s(\d\d):(\d\d):(\d\d)\s(.\d\d\d\d)\s/) { my $fildate = "$3-$4-$5-$6$7"; my $svndate = "{$3$4$5T$6$7Z}"; $revToDate{$1} = "$fildate\0$svndate\0$2"; } } close(F); my @keys = sort keys %revToDate; print STDERR "Checking out versions.\n"; foreach my $wgsrev (@keys) { my ($dirdate, $svndate, $author) = split '\0', $revToDate{$wgsrev}; #next if (-e "$wgsbyd/$dirdate"); #next if (-e "$wgsbyd/$dirdate.tar"); #next if (-e "$wgsbyd/$dirdate.tar.gz"); #next if (-e "$wgsbyd/$dirdate.tar.bz2"); #next if (-e "$wgsbyd/$dirdate.tar.xz"); next if ($dirdate eq "2004-04-14-1349"); # These are all one next if ($dirdate eq "2004-04-14-1350"); # slow commit, which next if ($dirdate eq "2004-04-14-1351"); # we capture in next if ($dirdate eq "2004-04-14-1352"); # 2004-04-14-1354 next if ($dirdate eq "2004-04-14-1353"); # next if ($dirdate le "2010-00"); next if ($dirdate eq "2011-09-06-1707"); # Doesn't compile next if ($dirdate eq "2012-02-01-2010"); # Broken GWS overlapStore next if ($dirdate eq "2012-02-01-2012"); # Broken GWS overlapStore next if ($dirdate eq "2012-02-01-2015"); # Broken GWS overlapStore next if ($dirdate le "2014-00"); system("mkdir -p $wgsbyd/$dirdate") if (! -e "$wgsbyd/$dirdate"); system("ln -s $dirdate $wgsbyd/$wgsrev") if (! -e "$wgsbyd/$wgsrev"); # Checkout the assembler if (! -e "$wgsbyd/$dirdate/src") { print "$dirdate -- $wgsrev CHECKOUT\n"; system("cd $wgsbyd/$dirdate ; svn co -r $wgsrev file://$wgssvn/trunk/src src > src.$wgsrev.err"); } else { print "$dirdate -- $wgsrev UPDATE\n"; system("cd $wgsbyd/$dirdate/src ; svn update -r $wgsrev"); } # Checkout kmer, or, more probably, just link to one that exists already my $kmerev = "0000"; open(L, "svn info -r \"$svndate\" file://$kmersvn/trunk |") or die "Failed to get info: $!\n"; while () { $kmerev = $1 if (m/^Revision:\s+(\d+)/); } close(L); die "Didn't find a kmer revision in $wgsbyd/$dirdate/kmer/kmer.err.\n" if ($kmerev eq "0000"); if (! -e "$wgsbyd/kmer$kmerev") { print "$dirdate -- kmer$kmerev CHECKOUT\n"; system("cd $wgsbyd/$dirdate && svn co -r \"$svndate\" file://$kmersvn/trunk $wgsbyd/kmer$kmerev > $wgsbyd/kmer$kmerev.err 2>&1"); } system("ln -s ../kmer$kmerev $wgsbyd/$dirdate/kmer") if (! -e "$wgsbyd/$dirdate/kmer"); # Compile if (! -e "$wgsbyd/kmer$kmerev/build.err") { print "$dirdate -- kmer$kmerev COMPILE\n"; if (! -e "$wgsbyd/kmer$kmerev/Makefile") { system("cd $wgsbyd/kmer$kmerev && sh configure.sh > configure.err 2>&1"); } open(F, "> $wgsbyd/kmer$kmerev/build.err"); print F "Waiting to compile.\n"; close(F); system("qsub -N kmer$kmerev -cwd -j y -o $wgsbyd/kmer$kmerev/build.err -q vomit.q -wd $wgsbyd/kmer$kmerev -b y gmake install"); #system("cd $wgsbyd/kmer$kmerev && gmake install > build.err 2>&1 &"); } if (! -e "$wgsbyd/$dirdate/src/build.err") { print "$dirdate -- $wgsrev COMPILE\n"; open(F, "> $wgsbyd/$dirdate/src/build.err"); print F "Waiting to compile.\n"; close(F); system("qsub -N wgs$wgsrev -hold_jid kmer$kmerev -cwd -j y -o $wgsbyd/$dirdate/src/build.err -q vomit.q -wd $wgsbyd/$dirdate/src -b y gmake"); #system("cd $wgsbyd/$dirdate/src && gmake > build.err 2>&1"); } # Archive #if (! -e "$wgsbyd/$dirdate.tar.bz2") { # Use sge, hold on both $kmerev and $wgsrev # system("cd $wgsbyd && tar -cf - $dirdate | xz -9vc > $dirdate.tar.bz2 && mv $wgsbyd/$dirdate $wgsbyd/$dirdate.DELETE &"); #} } canu-1.3/src/pipelines/sanity/compile-all-wgs-revisions.pl000066400000000000000000000203531272335422300237470ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; my $bgn = shift @ARGV; my $end = shift @ARGV; my @dates; open(F, "ls -r *bz2 |"); while () { chomp; next if (defined($bgn) && ($_ lt $bgn)); next if (defined($end) && ($end lt $_)); push @dates, $_; } $bgn = "START_OF_TIME" if (!defined($bgn)); $end = "END_OF_TIME" if (!defined($end)); print STDERR "Building ", scalar(@dates), " revisions from $bgn to $end.\n"; while (scalar(@dates) > 0) { my $bz2 = shift @dates; my $dir; if ($bz2 =~ m/(\d\d\d\d-\d\d-\d\d-\d\d\d\d).tar.bz2/) { $dir = $1; } else { die "Nope bz2='$bz2'\n"; } if (! -e "$dir") { print STDERR "Uncompressing $bz2\n"; system("bzip2 -dc $bz2 | tar -xf -") and die "Failed to uncompress '$bz2'\n"; next; } #if (! -e "$dir/kmer/FreeBSD-amd64") { # print STDERR "Building $dir/kmer\n"; # if ($dir le "2000-00-00-0000") { # die; # } else { # system("cd $dir/kmer && gmake install > errs 2>&1") and die "Failed to build '$dir/kmer'\n"; # } #} next if ($dir le "2008-10-10-0304"); # ALL the early assemblers fail to build; no kmer next if ($dir eq "2008-10-29-1053"); # buildRefUnitigs is not committed next if ($dir eq "2008-10-29-1619"); # buildRefUnitigs is not committed next if ($dir eq "2008-10-29-1649"); # buildRefUnitigs is not committed next if ($dir eq "2008-10-29-1651"); # buildRefUnitigs is not committed next if ($dir eq "2008-10-29-1719"); # buildRefUnitigs is not committed next if ($dir eq "2008-10-29-1720"); # buildRefUnitigs is not committed next if ($dir eq "2008-10-30-0449"); # buildRefUnitigs is not committed next if ($dir eq "2008-11-02-0628"); next if ($dir eq "2009-01-16-1639"); next if ($dir eq "2009-01-16-1643"); next if ($dir eq "2009-01-16-1646"); next if ($dir eq "2009-01-16-1647"); next if ($dir eq "2009-01-16-1657"); next if ($dir eq "2009-03-06-2012"); next if ($dir eq "2009-03-31-2004"); next if ($dir eq "2009-03-31-2032"); next if ($dir eq "2009-04-08-1725"); next if ($dir eq "2009-04-09-1401"); next if ($dir eq "2009-06-10-1805"); next if ($dir eq "2009-06-24-1205"); next if ($dir eq "2009-07-06-2003"); next if ($dir eq "2009-07-27-0806"); next if ($dir eq "2009-07-28-1230"); next if ($dir eq "2009-08-04-1103"); next if ($dir eq "2009-08-04-1105"); next if ($dir eq "2009-08-06-1136"); next if ($dir eq "2009-08-14-1103"); # BREAKS WITH rename-to-c++ next if ($dir eq "2009-08-28-17597"); next if ($dir eq "2009-09-04-2024"); next if ($dir eq "2009-09-04-2025"); next if ($dir eq "2009-09-07-0740"); next if ($dir eq "2009-09-09-0340"); next if ($dir eq "2009-09-09-0745"); next if ($dir eq "2009-10-12-0619"); next if ($dir eq "2009-10-27-1244"); next if ($dir eq "2009-11-19-1501"); next if ($dir eq "2009-12-03-0119"); next if ($dir eq "2009-12-19-0536"); next if ($dir eq "2010-01-14-0044"); next if ($dir eq "2010-01-26-0351"); # AS_BOG_BestOverlapGraph() prototype mismatch next if ($dir eq "2010-02-04-2156"); # BOG fails to build next if ($dir eq "2010-02-09-1812"); # markUniqueUnique() bad header file next if ($dir eq "2010-02-17-0132"); # pointer/object confusion next if ($dir eq "2010-03-22-2008"); # AS_UTL_writeFastQ() not defined next if ($dir eq "2010-09-28-1055"); # Makefile refers to missing overlapStore_genomeLength.c next if ($dir eq "2010-09-28-1720"); # Makefile refers to missing overlapStore_genomeLength.c next if ($dir eq "2010-10-01-1343"); next if ($dir eq "2010-10-04-0852"); next if ($dir eq "2010-10-15-0246"); # const char * vs char * in AS_BOG next if ($dir eq "2010-12-08-1242"); # Makefile refers to deleted AS_CGW/AS_CGW_dump.c next if ($dir eq "2010-12-08-1243"); # Makefile refers to deleted AS_CGW/AS_CGW_dump.c next if ($dir eq "2010-12-08-1245"); # Makefile refers to deleted AS_CGW/AS_CGW_dump.c next if ($dir eq "2010-12-09-0405"); # Makefile refers to deleted AS_CGW/AS_CGW_dump.c next if ($dir eq "2010-12-10-1328"); # Makefile refers to deleted AS_CGW/AS_CGW_dump.c next if ($dir eq "2011-01-25-1356"); next if ($dir eq "2011-02-11-0548"); next if ($dir eq "2011-03-08-2118"); # Undefined constant in AS_BOG next if ($dir eq "2011-06-17-1303"); next if ($dir eq "2011-08-01-1835"); next if ($dir eq "2011-08-01-2033"); next if ($dir eq "2011-08-01-2100"); next if ($dir eq "2011-08-01-2159"); next if ($dir eq "2011-08-01-2233"); next if ($dir eq "2011-08-02-0221"); next if ($dir eq "2011-08-02-0223"); next if ($dir eq "2011-08-02-0225"); next if ($dir eq "2011-08-02-0318"); next if ($dir eq "2011-08-04-1708"); next if ($dir eq "2011-08-24-2114"); next if ($dir eq "2011-08-25-0240"); next if ($dir eq "2011-08-30-1332"); next if ($dir eq "2011-08-30-1812"); next if ($dir eq "2011-08-30-1813"); next if ($dir eq "2011-08-30-2309"); next if ($dir eq "2011-09-02-1714"); # ouch, broken through 2011-09-06-1707 next if ($dir eq "2011-09-02-1825"); # ouch. next if ($dir eq "2011-09-02-2204"); # ouch. next if ($dir eq "2011-09-03-0129"); # ouch. next if ($dir eq "2011-09-03-0408"); # ouch. next if ($dir eq "2011-09-03-0736"); # ouch. next if ($dir eq "2011-09-03-0813"); # ouch. next if ($dir eq "2011-09-04-0101"); # ouch. next if ($dir eq "2011-09-05-1649"); # ouch. next if ($dir eq "2011-09-05-2123"); # ouch. next if ($dir eq "2011-09-06-0111"); # ouch. next if ($dir eq "2011-09-06-0215"); # ouch. next if ($dir eq "2011-09-06-0947"); # ouch. next if ($dir eq "2011-09-06-1415"); # ouch. next if ($dir eq "2011-09-06-1422"); # ouch. next if ($dir eq "2011-09-06-1506"); # ouch. next if ($dir eq "2011-09-06-1524"); # ouch. next if ($dir eq "2011-09-06-1527"); # ouch. next if ($dir eq "2011-09-06-1601"); # ouch. next if ($dir eq "2011-09-06-1641"); # ouch. next if ($dir eq "2011-09-06-1707"); # ouch. next if ($dir eq "2011-12-09-2302"); # Assigning 64-bit allocation to 32-bit pointer in AS_BAT if (! -e "$dir/kmer/FreeBSD-amd64") { die "Didn't find kmer in $dir\n"; } if (! -e "$dir/FreeBSD-amd64/bin/gatekeeper") { print STDERR "Building $dir/src\n"; if ($dir le "2009-02-03-2104") { # These old builds REQUIRE that SITE_NAME be set explicitly. $ENV{'SITE_NAME'} = "LOCAL"; system("cd $dir/src && gmake > errs 2>&1") and die "Failed to build '$dir/src'\n"; undef $ENV{'SITE_NAME'}; } elsif ($dir lt "2009-06-12-1741") { # These builds are before the C++ hack system("cd $dir/src && gmake > errs 2>&1") and die "Failed to build '$dir/src'\n"; } elsif ($dir le "2009-08-05-2205") { # These builds need to rename C to C++ to build. # The rename script doesn't go away until (after) 2009-09-30-1832 system("cd $dir/src && sh rename-to-c++.sh") and die "Failed to rename '$dir/src'\n"; system("cd $dir/src && gmake > errs 2>&1") and die "Failed to build '$dir/src'\n"; } else { system("cd $dir/src && gmake > errs 2>&1") and die "Failed to build '$dir/src'\n"; } } } canu-1.3/src/pipelines/sanity/sanity-all-done.pl000066400000000000000000000141251272335422300217340ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use MIME::QuotedPrint; use MIME::Base64; use HTML::Entities; use strict; if (scalar(@ARGV) != 5) { die "wrong args.\n"; } my $wrkdir = shift @ARGV; my $thisdate = shift @ARGV; my $label = shift @ARGV; my $addresses = shift @ARGV; my $assemblies = shift @ARGV; my @assemblies = split ',', $assemblies; my $timenow = localtime(); print "Subject: CAtest $thisdate $label\n"; print "To: $addresses\n"; print "Content-Type: multipart/mixed; boundary=Bri_Says_This_Is_The_Boundary\n"; # alternative if no atachments print "\n"; ######################################## # # Dump the HTML part # print "--Bri_Says_This_Is_The_Boundary\n"; print "Content-Type: text/html; charset=\"iso-8859-1\"\n"; print "Content-Transfer-Encoding: quoted-printable\n"; print "\n"; print "\n"; print "

\n"; print "" . encode_entities("Results for $thisdate (Finished at $timenow).") . "\n"; print "

\n"; print "\n"; print "

\n"; foreach my $asm (@assemblies) { open(R, "< $wrkdir/$thisdate/$asm/sanity-result.out"); $_ = ; close(R); chomp; # Green! s!SUCCESS!SUCCESS!g; s!same!same!g; # Red s!FAILURE!FAILURE!g; s!differs!differs!g; print "$_
\n"; } print "

\n"; print "
\n"; if ((! -e "$wrkdir/$thisdate/wgs/kmer.updates") || (-z "$wrkdir/$thisdate/wgs/kmer.updates")) { print "

\n"; print "No changes to kmer.\n"; print "

\n"; } else { print "

\n"; print "Changes for kmer.\n"; print "

\n"; print "

\n"; open(F, "< $wrkdir/$thisdate/wgs/kmer.updates"); while () { chomp; print encode_entities($_) . "
\n"; } close(F); print "

\n"; } print "
\n"; if ((! -e "$wrkdir/$thisdate/wgs/src.updates") || (-z "$wrkdir/$thisdate/wgs/src.updates")) { print "

\n"; print "No chages to wgs-assembler.\n"; print "

\n"; } else { print "

\n"; print "Changes for wgs-assembler.\n"; print "

\n"; print "

\n"; open(F, "< $wrkdir/$thisdate/wgs/src.updates"); while () { chomp; print encode_entities($_) . "
\n"; } close(F); print "

\n"; } #print "
\n"; #print "

\n"; #print "Build Results for kmer.\n"; #print "

\n"; #open(F, "< $wrkdir/$thisdate/wgs/kmer/make.err"); #while () { # chomp; # print encode_entities($_) . "
\n"; #} #close(F); #print "
\n"; #print "

\n"; #print "Build Results for wgs-assembler.\n"; #print "

\n"; #open(F, "< $wrkdir/$thisdate/wgs/src/make.err"); #while () { # chomp; # print encode_entities($_) . "
\n"; #} #close(F); print "\n"; print "\n"; ######################################## # # Dump the ASCII part -- not dumped; outlook shows this as another attachment # #print "--Bri_Says_This_Is_The_Boundary\n"; #print "Content-Type: text/plain; charset=\"iso-8859-1\"\n"; #print "Content-Transfer-Encoding: quoted-printable\n"; #print "\n"; ######################################## # # Dump the checkout and build results # attachFile("$wrkdir/$thisdate/wgs/kmer.updates", "kmerChanges.txt"); attachFile("$wrkdir/$thisdate/wgs/src.updates", "wgsChanges.txt"); attachFile("$wrkdir/$thisdate/wgs/kmer/make.err", "kmerBuildError.txt"); attachFile("$wrkdir/$thisdate/wgs/src/make.err", "wgsBuildError.txt"); ######################################## # # Dump the QC attachments # foreach my $asm (@assemblies) { my $errfile = "$wrkdir/$thisdate/$asm/sanity-error.out"; my $qcfile = "$wrkdir/$thisdate/$asm/sanity-qc.out"; if ((-e $qcfile) && (! -z $qcfile)) { my @n = split "/", $qcfile; my $l = scalar(@n); my $name = "$n[$l-3]-$n[$l-2]-qc.txt"; attachFile($qcfile, $name); } if ((-e $errfile) && (! -z $errfile)) { my @n = split "/", $errfile; my $l = scalar(@n); my $name = "$n[$l-3]-$n[$l-2]-error.txt"; attachFile($errfile, $name); } } ######################################## # # All done. DO NOT add another boundary - this results in a bogus empty attachment. # #print "--Bri_Says_This_Is_The_Boundary\n"; sub attachFile ($$) { my $fileName = shift @_; # Name of file on disk my $attachName = shift @_; # Name of file in email return if (! -e $fileName); return if (-z $fileName); print "--Bri_Says_This_Is_The_Boundary\n"; print "Content-Type: application/octet-stream; name=\"$attachName\"\n"; print "Content-Transfer-Encoding: base64\n"; print "Content-Disposition: attachment; filename=\"$attachName\"\n"; print "\n"; my @message; my $message; # Winblows wants \r\n, and since that's the primary consumer of these emails, we put it in. open(F, "< $fileName"); while () { chomp; push @message, "$_\r\n"; } close(F); $message = join "", @message; local($/) = undef; print encode_base64($message); } canu-1.3/src/pipelines/sanity/sanity-asm-done.pl000077500000000000000000000074441272335422300217550ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## if (scalar(@ARGV) != 3) { die "wrong args.\n"; } my $wrkdir = shift @ARGV; my $prefix = shift @ARGV; my $thisdate = shift @ARGV; # Special hack to add group rw system("chmod -R ug+rw $wrkdir/$thisdate/$prefix"); system("chgrp -R atg $wrkdir/$thisdate/$prefix"); my $lastdate = undef; my $gooddate = undef; my $resultlast = "none"; my $resultgood = "none"; my $diffs = "$wrkdir/$thisdate/$prefix/9-terminator/$prefix.qc"; # Special case; don't set last/ref to the current date. Useful in # testing (and restarting in general). if (-e "$wrkdir/POINTERS/$prefix.last") { open(F, "< $wrkdir/POINTERS/$prefix.last"); while () { chomp; $lastdate = $_ if ($_ lt $thisdate); } close(F); } if (-e "$wrkdir/POINTERS/$prefix.reference") { open(F, "< $wrkdir/POINTERS/$prefix.reference"); while () { chomp; $gooddate = $_ if ($_ lt $thisdate); } close(F); } open(RESULT, "> $wrkdir/$thisdate/$prefix/sanity-result.out") or die; open(ERROR, "> $wrkdir/$thisdate/$prefix/sanity-error.out") or die; if (! -e "$wrkdir/$thisdate/$prefix/9-terminator/$prefix.asm") { print RESULT "Assembly result for $thisdate/$prefix: FAILURE (no asm file)\n"; open(F, "ls -ltr $wrkdir/$thisdate/$prefix |"); while () { print ERROR $_; } close(F); print ERROR "\n"; print ERROR "\n"; print ERROR "\n"; my $logfile; open(F, "ls $wrkdir/$thisdate/$prefix/runCA.sge.out.[0-9][0-9] |"); while () { chomp; $logfile = $_; } close(F); open(F, "< $logfile"); while () { print ERROR $_; } close(F); close(RESULT); close(ERROR); exit(0); } else { if (defined($lastdate) && (-e "$wrkdir/$lastdate/$prefix/9-terminator/$prefix.asm")) { $diffs = "$wrkdir/$lastdate/$prefix/9-terminator/$prefix.qc $diffs"; if (system("diff -q $wrkdir/$thisdate/$prefix/9-terminator/$prefix.asm $wrkdir/$lastdate/$prefix/9-terminator/$prefix.asm > /dev/null 2>&1") == 0) { $resultlast = "$lastdate same"; } else { $resultlast = "$lastdate differs"; } } if (defined($gooddate) && (-e "$wrkdir/$gooddate/$prefix/9-terminator/$prefix.asm")) { $diffs = "$wrkdir/$gooddate/$prefix/9-terminator/$prefix.qc $diffs"; if (system("diff -q $wrkdir/$thisdate/$prefix/9-terminator/$prefix.asm $wrkdir/$gooddate/$prefix/9-terminator/$prefix.asm > /dev/null 2>&1") == 0) { $resultgood = "$gooddate same"; } else { $resultgood = "$gooddate differs"; } } print RESULT "Assembly result for $thisdate/$prefix: SUCCESS (last: $resultlast) (reference: $resultgood)\n"; system("perl $wrkdir/sanity-merge-qc.pl $diffs > $wrkdir/$thisdate/$prefix/sanity-qc.out"); open(F, ">> $wrkdir/POINTERS/$prefix.last"); print F "$thisdate\n"; close(F); } close(RESULT); close(ERROR); canu-1.3/src/pipelines/sanity/sanity-get-next-date.pl000066400000000000000000000031441272335422300227060ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use Time::Local; my @v; my $date = shift @ARGV; my $offt = shift @ARGV; my $type = shift @ARGV; my $time = time(); if ($date =~ m/(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d)(\d\d)/) { $time = timelocal(0, $5, $4, $3, $2 - 1, $1); } $time += $offt; my @v = localtime($time); $v[5] += 1900; $v[4]++; $v[5] = substr("0000$v[5]", -4); $v[4] = substr("0000$v[4]", -2); $v[3] = substr("0000$v[3]", -2); $v[2] = substr("0000$v[2]", -2); $v[1] = substr("0000$v[1]", -2); #$v[2] = "00"; #$v[1] = "01"; if ($type eq "next") { $thisdate = "$v[5]-$v[4]-$v[3]-$v[2]$v[1]"; } elsif ($type eq "hold") { $thisdate = "$v[5]$v[4]$v[3]$v[2]$v[1].00"; } else { $thisdate = "$v[4]$v[3]-$v[2]$v[1]"; } print "$thisdate\n"; canu-1.3/src/pipelines/sanity/sanity-merge-qc.pl000066400000000000000000000056551272335422300217510ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Simple hack to merge multiple QC reports into one. Uses the first # QC on the command line to figure out what fields exist, then writes # a multiple column QC using those fields and data from all QC files # on the command line. my @labels; my %values; my $s; my $l; my $v; my $firstFile = 1; while (scalar(@ARGV) > 0) { if (open(F, "< $ARGV[0]")) { if ($ARGV[0] =~ m!.*/NIGHTLY/(\d\d\d\d-\d\d-\d\d-\d\d\d\d)/(.*)/9-terminator/.*.qc$!) { print "$1/$2\n"; } else { print "$ARGV[0]\n"; } while () { $_ =~ s/^\s+//; $_ =~ s/\s+$//; if ($_ =~ m/^\s+$/) { next; } if ($_ =~ m/^\[(.*)\]$/) { $s = $1; next; } if ($_ =~ m/^(.*)=(.*)$/) { $l = "$s\0$1"; $v = $2; } else { next; } if ($firstFile) { push @labels, $l; } $values{$l} .= substr(" $v ", 0, 16); $values{$l} .= "BRIWASHERE"; } close(F); my @k = keys %values; foreach my $l (@k) { if ($values{$l} =~ m/^(.*)BRIWASHERE$/) { $values{$l} = $1; } else { $values{$l} .= substr(" N/A ", 0, 16); } } $firstFile = 0; } shift @ARGV; } my $lasts; foreach my $xx (@labels) { ($s, $l) = split '\0', $xx; if ($s ne $lasts) { print "\n[$s]\n"; $lasts = $s; } $l = substr("$l ", 0, 20); my $d = " "; my @v = split '\s+', $values{$xx}; foreach my $a (@v) { foreach my $b (@v) { # Because $xx is whitespace justified, the first # thing from the split can be empty. if (($a ne "") && ($b ne "") && ($a ne $b)) { $d = " ** "; } } } print "$l$d$values{$xx}\n"; } canu-1.3/src/pipelines/sanity/sanity-purge-old.pl000066400000000000000000000122461272335422300221410ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Purge all but the last reference assembly, and all but the last two # assemblies, and all but the last two that failed to complete. # # Up to five assemblies are saved: # The reference # Two crashes # Two successful finished assemblies # # No crashes are save that are earlier than the latest successful assembly. # # OLDEST # crash # reference (saved) # crash # finished # finished # finished (saved) # crash # crash # finished (saved) # crash (saved) # NEWEST my @assemblies; my @nightlies; my $doPurge = ($ARGV[0] eq "purge"); # This is automagically updated by sanity-asm-done.pl, whenever an assembly finishes. open(F, "ls POINTERS/*last |"); while () { if ($_ =~ m/^POINTERS\/(.*).last$/) { push @assemblies, $1; } } open(F, "ls -d ????-??-??-???? |"); while () { chomp; push @nightlies, $_; } close(F); foreach my $asm (@assemblies) { my $reference; my $last1; my $last2; my $crash1; my $crash2; # Read reference pointers, find the latest. { my %ref; my @ref; $ref{"0000-00-00-0000"}++; if (! -e "POINTERS/$asm.reference") { open(F, "> POINTERS/$asm.reference"); close(F); } open(F, "< POINTERS/$asm.reference") or die; while () { chomp; if (-d "$_/$asm") { $ref{$_}++; } } close(F); @ref = sort keys %ref; $reference = pop @ref; } # Read success pointers, find the latest two. { my %ref; my @ref; $ref{"0000-00-00-0000"}++; $ref{"0000-00-00-0001"}++; if (! -e "POINTERS/$asm.last") { open(F, "> POINTERS/$asm.last"); close(F); } open(F, "< POINTERS/$asm.last") or die; while () { chomp; if (-e "$_/$asm/$asm.qc") { $ref{$_}++; } } close(F); @ref = sort keys %ref; $last1 = pop @ref; $last2 = pop @ref; } # Find the last two failures. { my %ref; my @ref; $ref{"0000-00-00-0000"}++; $ref{"0000-00-00-0001"}++; foreach my $n (@nightlies) { if (! -e "$n/$asm/$asm.qc") { $ref{$n}++; } } @ref = sort keys %ref; $crash1 = pop @ref; $crash2 = pop @ref; $crash1 = "0000-00-00-0000" if ($crash1 lt $last1); $crash2 = "0000-00-00-0000" if ($crash2 lt $last1); } print STDERR "REF\t$reference\tLAST\t$last1\t$last2\tCRASH\t$crash1\t$crash2\tASM\t$asm\n"; # Save the last TWO finished assemblies, or just one? Comment to save two. $last2 = "0000-00-00-0000"; foreach my $n (@nightlies) { if (-d "$n/$asm") { my $finished = (-e "$n/$asm/$asm.qc") ? "FINISHED" : "CRASHED"; $finished = ($n eq $reference) ? "REFERENCE" : $finished; if (($n ne $reference) && ($n ne $last2) && ($n ne $last1) && ($n ne $crash2) && ($n ne $crash1)) { print STDERR "REMOVE\t$n/$asm\n"; if ($doPurge) { if (! -d "DEL") { system("mkdir DEL"); } if (! -d "DEL/$n") { system("mkdir DEL/$n"); } # Save some juicy bits rename "$n/$asm/9-terminator/$asm.qc", "$n/$asm.qc"; rename "$n/$asm/4-unitigger/$asm.cga.0", "$n/$asm.cga.0"; rename "$n/$asm", "DEL/$n/$asm"; } } else { print STDERR "SAVE\t$n/$asm\n"; } } } } foreach my $dir (@nightlies) { my $asmExist = 0; next if (! -d "$dir/wgs"); foreach my $asm (@assemblies) { $asmExist++ if (-d "$dir/$asm"); } if ($asmExist == 0) { print STDERR "$dir has $asmExist saved assemblies; purge source code.\n"; if ($doPurge) { if (! -d "DEL") { system("mkdir DEL"); } if (! -d "DEL/$dir") { system("mkdir DEL/$dir"); } rename "$dir/wgs", "DEL/$dir/wgs"; } } else { print STDERR "$dir has $asmExist saved assemblies.\n"; } } canu-1.3/src/pipelines/sanity/sanity-update-reference.pl000066400000000000000000000043531272335422300234610ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; # Updates the reference POINTER to the latest successful assembly. my @assemblies; open(F, "ls POINTERS/*last |"); while () { if ($_ =~ m/^POINTERS\/(.*).last$/) { push @assemblies, $1; } } foreach my $asm (@assemblies) { my $reference = "0000-00-00-0000"; my $last = "0000-00-00-0000"; if (-e "POINTERS/$asm.reference") { my %ref; my @ref; $ref{"0000-00-00-0000"}++; open(F, "< POINTERS/$asm.reference") or die; while () { chomp; if (-d "$_/$asm") { $ref{$_}++; } } close(F); @ref = sort keys %ref; $reference = pop @ref; } if (-e "POINTERS/$asm.last") { my %ref; my @ref; $ref{"0000-00-00-0000"}++; open(F, "< POINTERS/$asm.last") or die; while () { chomp; if (-e "$_/$asm/$asm.qc") { $ref{$_}++; } } close(F); @ref = sort keys %ref; $last = pop @ref; } if ($reference lt $last) { print STDERR "REF\t$reference\tLAST\t$last\tSTALE\t$asm\n"; open(F, ">> POINTERS/$asm.reference") or die; print F "$last\n"; close(F); } else { print STDERR "REF\t$reference\tLAST\t$last\t\t$asm\n"; } } canu-1.3/src/pipelines/sanity/sanity.pl000077500000000000000000000366631272335422300202610ustar00rootroot00000000000000#!/usr/bin/env perl ############################################################################### # # This file is part of canu, a software program that assembles whole-genome # sequencing reads into contigs. # # This software is based on: # 'Celera Assembler' (http://wgs-assembler.sourceforge.net) # the 'kmer package' (http://kmer.sourceforge.net) # both originally distributed by Applera Corporation under the GNU General # Public License, version 2. # # Canu branched from Celera Assembler at its revision 4587. # Canu branched from the kmer project at its revision 1994. # # Modifications by: # # Brian P. Walenz beginning on 2015-OCT-12 # are a 'United States Government Work', and # are released in the public domain # # File 'README.licenses' in the root directory of this distribution contains # full conditions and disclaimers for each license. ## use strict; use Config; # for @signame # The only four globals configurables: # my $site = undef; my $wrkdir = undef; my $wgssvn = undef; my $kmersvn = undef; if (-d "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY") { $site = "JCVI"; $wrkdir = "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY"; $wgssvn = "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY/SVN-wgs"; $kmersvn = "/usr/local/projects/BIOINFO/ASSEMBLY/NIGHTLY/SVN-kmer"; } elsif (-d "/work/NIGHTLY/") { $site = "BPWI"; $wrkdir = "/work/NIGHTLY"; $wgssvn = "/work/NIGHTLY/SVN-wgs"; $kmersvn = "/work/NIGHTLY/SVN-kmer"; } else { die "Unknown site configuration.\n"; } # Command line options are # # 'oper' -- thing to do. # # 'ddir' -- properly formatted directory name to do it in. op == # checkout creates this directory. If still supplied, it will # force a checkout from that date. # # rsync # update the local repositories in 'wgs-assembler-cvs' and 'kmer-svn'. # # checkout date # Date must be of the form yyyy-mm-dd-hhmm. This will checkout # the assembler current at that time. If not present, the current # time is assumed. A change log is generated from the last # date present. # # build date # Builds the assembler checked out into 'date'. # # assemble date label specFile .... email ... # Launches runCA on each specFile, running it in directory 'date'. # Each specFile must have a unique name; the assembly is named # after the specFile. The label is used only for reporting via email. # At the end, a diff is generated to POINTERS/$prefix.last and # POINTERS/$prefix.reference. If the assembly finished, POINTERS/$prefix.last # is updated. # { my $oper = shift @ARGV; my $ddir = shift @ARGV; my ($thisdate, $lastdate) = parseDate($ddir); if ($oper eq "rsync") { system("mkdir -p $wgssvn") if (! -d "$wgssvn"); system("mkdir -p $kmersvn") if (! -d "$kmersvn"); #system("cd $wgscvs && rsync -av rsync://wgs-assembler.cvs.sourceforge.net/cvsroot/wgs-assembler/\* . > rsync.out 2>&1"); system("cd $wgssvn && rsync -av svn.code.sf.net::p/wgs-assembler/svn/ . > rsync.out 2>&1"); system("cd $kmersvn && rsync -av svn.code.sf.net::p/kmer/code/ . > rsync.out 2>&1"); } elsif ($oper eq "checkout") { checkoutAndLog($thisdate, $lastdate, "file://$kmersvn/trunk", "kmer", "kmer"); checkoutAndLog($thisdate, $lastdate, "file://$wgssvn/trunk/src", "src", "src"); } elsif ($oper eq "build") { buildKmer($thisdate); buildCA($thisdate); } elsif ($oper eq "assemble") { assemble($thisdate, @ARGV); } else { die "$0: unknown action '$oper'\n"; } exit(0); } sub parseDate ($) { my $pathdate = shift @_; my $thisdate; my $lastdate; # Until our Makefile is fixed, the ":" cannot be used, and so we need # to adopt a slightly goofy timestamp name. if (defined($pathdate)) { if ($pathdate =~ m/^\d\d\d\d-\d\d-\d\d-\d\d\d\d$/) { $thisdate = $pathdate; } else { die "Malformed ddir '$pathdate'\n"; } } else { my @v = localtime; $v[5] += 1900; $v[4]++; $v[5] = substr("0000$v[5]", -4); $v[4] = substr("0000$v[4]", -2); $v[3] = substr("0000$v[3]", -2); $v[2] = substr("0000$v[2]", -2); $v[1] = substr("0000$v[1]", -2); $thisdate = "$v[5]-$v[4]-$v[3]-$v[2]$v[1]"; } open(F, "ls $wrkdir |"); while () { chomp; if (m/^\d\d\d\d-\d\d-\d\d-\d\d\d\d$/) { if ($_ lt $thisdate) { $lastdate = $_; } } } close(F); print STDERR "Working on '$thisdate' -- last found is '$lastdate'.\n"; return($thisdate, $lastdate); } sub checkoutAndLog ($$) { my $thisdate = shift @_; my $lastdate = shift @_; my $repo = shift @_; # Path to local repository my $target = shift @_; # Thing to checkout my $path = shift @_; # Where to put it (in wgs/) print STDERR "Checking out $wrkdir/$thisdate (repo=$repo path=$path)\n"; if (-d "$wrkdir/$thisdate/wgs/$path") { print STDERR "$wrkdir/$thisdate/wgs/$path already exists. Please remove to rerun.\n"; return; } system("mkdir -p $wrkdir/$thisdate/wgs") if (! -d "$wrkdir/$thisdate/wgs"); # Convert time-names to dates that svn can use. my $thisdatesvn; my $lastdatesvn; # NOT tested with SVN. The old format was "-04:00"; new format is "-0400". my $tz = `date +%z`; chomp $tz; if ($thisdate =~ m/(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d)(\d\d)/) { $thisdatesvn = "$1-$2-$3T$4:$5$tz"; } else { } if (defined($lastdate)) { if ($lastdate =~ m/(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d)(\d\d)/) { $lastdatesvn = "$1-$2-$3T$4:$5$tz"; } else { } } system("cd $wrkdir/$thisdate/wgs && svn co -r \"{$thisdatesvn}\" $repo $target > $path.checkout.err 2>&1"); # This is annoying. SVN log will report changes inclusive to revisions. -r 5:9 will report # changes made in revisions 5 through 9. When you give it a date, it finds the revision that # was active on that date. What we want here, though, is the changes SINCE that date (or, # since revision 5, up to revision 9). # # To get around this, we first get the logs, but scan for the lowest and highest revision # numbers, then dump correctly. # if ($lastdate ne "") { my $loRev; my $hiRev; print "svn log -v $repo -r \"{$lastdatesvn}:{$thisdatesvn}\"\n"; open(F, "cd $wrkdir/$thisdate/wgs && svn log -v $repo -r \"{$lastdatesvn}:{$thisdatesvn}\" |"); while () { if (m/^r(\d+)\s+\|\s+/) { $loRev = $1 if ((!defined($loRev)) || ($1 < $loRev)); $hiRev = $1 if ((!defined($hiRev)) || ($hiRev < $1)); } } close(F); $loRev++ if (defined($loRev)); print STDERR "loRev='$loRev' hiRev='$hiRev'\n"; if (defined($loRev) && defined($hiRev) && ($loRev < $hiRev)) { print "svn log -v $repo -r $loRev:$hiRev\n"; system("cd $wrkdir/$thisdate/wgs && svn log -v $repo -r $loRev:$hiRev > $path.updates"); } } print STDERR "$thisdate checked out!\n"; } sub buildKmer ($) { my $thisdate = shift @_; print STDERR "Building KMER $wrkdir/$thisdate\n"; if (-e "$wrkdir/$thisdate/wgs/kmer/make.err") { print STDERR "$wrkdir/$thisdate/wgs/kmer was already built once (successuflly or not). Please cleanup first.\n"; } else { system("cd $wrkdir/$thisdate/wgs/kmer && sh configure.sh > configure.out 2> configure.err"); system("cd $wrkdir/$thisdate/wgs/kmer && gmake install > make.out.raw 2> make.err.raw"); my %lines; open(F, "< $wrkdir/$thisdate/wgs/kmer/make.err.raw"); open(G, "> $wrkdir/$thisdate/wgs/kmer/make.err"); while () { chomp; next if (m/^ar:\s+creating/); if (!exists($lines{$_})) { $lines{$_}++; print G "$_\n"; } } close(F); close(G); } } sub buildCA ($) { my $thisdate = shift @_; print STDERR "Building CA $wrkdir/$thisdate\n"; if (-e "$wrkdir/$thisdate/wgs/src/make.err") { print STDERR "$wrkdir/$thisdate/wgs/src was already built once (successuflly or not). Please cleanup first.\n"; } else { # Temporary hack to handle the C to C++ renaming. # This was broken on "2009/06/10 17:34:22". # This was fixed on "2009/08/06 11:36:54" # if ((-e "$wrkdir/$thisdate/wgs/src/rename-to-c++.sh") && (-s "$wrkdir/$thisdate/wgs/src/c_make.gen" == 14877)) { system("cd $wrkdir/$thisdate/wgs/src && sh rename-to-c++.sh"); } system("cd $wrkdir/$thisdate/wgs/src && gmake > make.out.raw 2> make.err.raw"); my %lines; open(F, "< $wrkdir/$thisdate/wgs/src/make.err.raw"); open(G, "> $wrkdir/$thisdate/wgs/src/make.err"); while () { chomp; next if (m/^ar:\s+creating/); if (!exists($lines{$_})) { $lines{$_}++; print G "$_\n"; } } close(F); close(G); } } sub assemble ($$@) { my $thisdate = shift @_; my $label = shift @_; my $holds_asms; my $names_asms; my $holds_done; my $arch; # Ripped from runCA/util.pl my $syst = `uname -s`; chomp $syst; # OS implementation my $arch = `uname -m`; chomp $arch; # Hardware platform my $name = `uname -n`; chomp $name; # Name of the system $arch = "amd64" if ($arch eq "x86_64"); $arch = "ppc" if ($arch eq "Power Macintosh"); # end of rip. # Figure out which are specfiles and which are email addresses. # my @spec; my @addresses; my $addresses; foreach my $arg (@_) { if ($arg =~ m/\@/) { push @addresses, $arg; } else { $arg = "$ENV{PWD}/$arg" if ($arg !~ m/^\//); push @spec, $arg; } } $addresses = join ',', @addresses; open(F, "> $wrkdir/$thisdate/asm-done.sh"); print F "#!/bin/sh\n"; print F "\n"; print F "perl $wrkdir/sanity-asm-done.pl $wrkdir \$1 $thisdate\n"; close(F); open(F, "> $wrkdir/$thisdate/all-done.sh"); print F "#!/bin/sh\n"; print F "\n"; print F "perl $wrkdir/sanity-all-done.pl $wrkdir $thisdate \$1 $addresses \$2 \\\n"; print F "| \\\n"; print F "tee \"$wrkdir/$thisdate/sanity-all-done.\$1\" \\\n"; print F "| \\\n"; print F "/usr/sbin/sendmail -i -t -f celera_assembler_test\@jcvi.org\n" if ($site eq "JCVI"); print F "/usr/local/sbin/ssmtp thebri\@gmail.com\n" if ($site eq "BPWI"); close(F); foreach my $s (@spec) { my $n; { my @c = split '/', $s; $n = $c[scalar(@c) - 1]; $n =~ s/.spec$//; $n =~ s/.specFile$//; } print STDERR "----------------------------------------\n"; print STDERR "Submitting assembly '$n'.\n"; system("mkdir $wrkdir/$thisdate/$n") if (! -d "$wrkdir/$thisdate/$n"); my $jl = "CAini_${n}_$$"; # Name of the launcher my $jn = "CAtst_${n}_$$"; # Name of the asm-done open(F, "> $wrkdir/$thisdate/$n/launch-assembly.sh"); print F "#!/bin/sh\n"; print F "\n"; print F "# runCA checks if this is set to decide if it is on the grid or not. We want\n"; print F "# to pretend we are NOT on the grid, so runCA will resubmit itself immediately.\n"; print F "#\n"; print F "unset SGE_TASK_ID\n"; print F "\n"; print F "\n"; print F "# Attempt to (re)configure SGE. For reasons Bri doesn't know,\n"; print F "# jobs submitted to SGE, and running under SGE, fail to read his\n"; print F "# .tcshrc (or .bashrc, limited testing), and so they don't setup\n"; print F "# SGE (or ANY other paths, etc) properly. For the record,\n"; print F "# interactive SGE logins (qlogin, etc) DO set the environment.\n"; print F "\n"; print F ". \$SGE_ROOT/\$SGE_CELL/common/settings.sh\n"; print F "\n"; print F "\n"; print F "# Submit runCA to the grid. Do not set any runCA parameters here; they\n"; print F "# will override ALL specFile parameters. scriptOnGrid must be set, otherwise\n"; print F "# the assembly will be performed with this call, instead of just launched\n"; print F "# to the grid.\n"; print F "\n"; print F "perl $wrkdir/$thisdate/wgs/$syst-$arch/bin/runCA \\\n"; print F " sgePropagateHold=$jn scriptOnGrid=1 \\\n"; print F " -p $n -d $wrkdir/$thisdate/$n -s $s\n"; print F "\n"; print F "# Once that runCA finishes, we've updated the hold on $jn, and so can release\n"; print F "# our user hold on it.\n"; print F "#\n"; print F "qrls -h u $jn\n"; print F "\n"; print F "\n"; close(F); # A separate script is used to launch runCA, so that we can get a user hold on it. This is messy. # 1) Submit the launcher, holding it. # 2) Submit the asm-done, holding it too. Make it also hold_jid on the launcher. # 3) Release the launcher. # 4) The launcher submits runCA, with sgePropagateHold. When that runCA finishes, the assembly # is now running, AND it has updated the hold_jid on asm-done. # 5) We can now release the asm-done. # # Steps 1 and 2 are done here, step 3 is at the very end, steps 4 and 5 are done in the launch-assembly.sh above. if ($site eq "JCVI") { system("cd $wrkdir/$thisdate/$n && qsub -P 334007 -A CAsanity -b n -cwd -j y -o $wrkdir/$thisdate/$n/launch-assembly.err -l fast -h -N $jl $wrkdir/$thisdate/$n/launch-assembly.sh"); system("cd $wrkdir/$thisdate/$n && qsub -P 334007 -A CAsanity -b n -cwd -j y -o $wrkdir/$thisdate/$n/asm-done.err -l fast -h -hold_jid $jl -N $jn $wrkdir/$thisdate/asm-done.sh $n"); } elsif ($site eq "BPWI") { system("cd $wrkdir/$thisdate/$n && qsub -A CAsanity -b n -cwd -j y -o $wrkdir/$thisdate/$n/launch-assembly.err -h -N $jl $wrkdir/$thisdate/$n/launch-assembly.sh"); system("cd $wrkdir/$thisdate/$n && qsub -A CAsanity -b n -cwd -j y -o $wrkdir/$thisdate/$n/asm-done.err -h -hold_jid $jl -N $jn $wrkdir/$thisdate/asm-done.sh $n"); } if (defined($holds_done)) { $holds_done .= ",$jn"; } else { $holds_done = "$jn"; } if (defined($holds_asms)) { $holds_asms .= ",$jl"; $names_asms .= ",$n"; } else { $holds_asms = "$jl"; $names_asms .= "$n"; } } if ($site eq "JCVI") { system("cd $wrkdir/$thisdate && qsub -P 334007 -A CAsanity -b n -cwd -j y -o $wrkdir/$thisdate/all-done.err -l fast -hold_jid $holds_done -N CAfin_$thisdate $wrkdir/$thisdate/all-done.sh $label $names_asms"); } elsif ($site eq "BPWI") { system("cd $wrkdir/$thisdate && qsub -A CAsanity -b n -cwd -j y -o $wrkdir/$thisdate/all-done.err -hold_jid $holds_done -N CAfin_$thisdate $wrkdir/$thisdate/all-done.sh $label $names_asms"); } # Now, release everything to run. system("qrls -h u $holds_asms"); } exit(0); canu-1.3/src/pipelines/sanity/sanity.sh000066400000000000000000000047131272335422300202440ustar00rootroot00000000000000#!/bin/sh # (re)Load the sge config. . $SGE_ROOT/$SGE_CELL/common/settings.sh # Needed for old checkouts. Newer checkouts default to LOCAL. #With SITE=JCVI, every EUID is different, so every QC report is different. #With default behavior, EUIDs always start at the same number, so QC reports can be identical. #We want default bahavior so the summary email reports no differences. #export SITE_NAME=JCVI # # Master controller of nightly sanity checks. Optional date on command line. # # To start the next nightly, launch with something like: # qsub -cwd -j y -o init.err -A assembly-nightly -l fast -a 200907180001.00 -b n sanity.sh 2009-07-18-0001 grid # # To start a previous nightly, the sge hold isn't needed, well, neither is sge for that matter. # Just run sanity.sh with the date/time you want to start at. # # CAREFUL! Submitting something in the past will automagically submit every nightly since that date! # date=$1 grid=$2 if [ x$date != x ] ; then echo "SANITY BEGINS for $date at `date`" fi # Remove old versions perl sanity-purge-old.pl purge rm -rf DEL # Update the repository. perl sanity.pl rsync # Checkout the latest version. perl sanity.pl checkout $date # Figure out what was checked out. if [ x$date = x ] ; then date=`ls -1d 20??-??-??-???? | tail -n 1` fi # Build it. perl sanity.pl build $date # Let the user pick one to run if [ x$grid = x ] ; then echo "$date checked out and compiled. Run some of:" echo " sh sanity-daily-test.sh $date" echo " sh sanity-daily-pging.sh $date" else nextofft=604800 # one week nextofft=86400 # one day nextofft=14400 # four hours nextofft=7200 # two hours nextofft=21600 # six hours nextofft=43200 # twelve hours nextdate=`perl sanity-get-next-date.pl $date $nextofft next` nexthold=`perl sanity-get-next-date.pl $date $nextofft hold` nextname=`perl sanity-get-next-date.pl $date $nextofft name` # Which tests should we run? For now, just the simple p.ging sh sanity-daily-pging.sh $date #if [ `date +%u` = 6] ; then # sh sanity-weekly-dros.sh $date # sh sanity-weekly-moore.sh $date #fi echo "SUBMIT for $nextdate ($nexthold)" echo \ qsub -cwd -j y -o $nextdate.err -P 334007 -A assembly-nightly -N CAsnty$nextname -a $nexthold -b n sanity.sh $nextdate grid qsub -cwd -j y -o $nextdate.err -P 334007 -A assembly-nightly -N CAsnty$nextname -a $nexthold -b n sanity.sh $nextdate grid fi canu-1.3/src/stores/000077500000000000000000000000001272335422300144145ustar00rootroot00000000000000canu-1.3/src/stores/gatekeeperCreate.C000066400000000000000000000670541272335422300177740ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_GKP/gkpStoreCreate.C * * Modifications by: * * Brian P. Walenz from 2012-FEB-06 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-OCT-09 to 2015-AUG-10 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-08 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "findKeyAndValue.H" #include "AS_UTL_fileIO.H" #undef UPCASE // Don't convert lowercase to uppercase, special case for testing alignments. #define UPCASE // Convert lowercase to uppercase. Probably needed. // Canu doesn't really use QVs as of late 2015. Only consensus is aware of them, but since read // correction doesn't output QVs, the QVs that consensus sees are totally bogus. So, disable // storage of them - gatekeeperCreate will store a default QV for every read, and consensus will // process as usual. // #define DO_NOT_STORE_QVs // Support fastq of fasta, even in the same file. // Eventually want to support bax.h5 natively. uint32 validSeq[256] = {0}; uint32 loadFASTA(char *L, char *H, char *S, uint32 &Slen, char *Q, compressedFileReader *F, FILE *errorLog, uint32 &nWARNS) { uint32 nLines = 0; // Lines read from the input uint32 nBases = 0; // Bases read from the input, used for reporting errors bool valid = true; // We've already read the header. It's in L. But we want to use L to load the sequence, so the // header is copied to H. We need to do a copy anyway - we need to return the next header in L. // We can either copy L to H now, or use H to read lines and copy H to L at the end. We're stuck // either way. strcpy(H, L + 1); // Clear the sequence. S[0] = 0; Q[0] = 0; // Sentinel to tell gatekeeper to use the fixed QV value Slen = 0; // Load sequence. This is a bit tricky, since we need to peek ahead // and stop reading before the next header is loaded. Instead, we read the // next line into what we'd read the header into outside here. fgets(L, AS_MAX_READLEN+1, F->file()); nLines++; chomp(L); // Catch empty reads - reads with no sequence line at all. if (L[0] == '>') { fprintf(errorLog, "read '%s' is empty.\n", H); nWARNS++; return(nLines); } // Copy in the sequence, as long as it is valid sequence. If any invalid letters // are found, set the base to 'N'. uint32 baseErrors = 0; while ((!feof(F->file())) && (L[0] != '>')) { nBases += strlen(L); // Could do this in the loop below, but it makes it ugly. for (uint32 i=0; (Slen < AS_MAX_READLEN) && (L[i] != 0); i++) { switch (L[i]) { #ifdef UPCASE case 'a': S[Slen] = 'A'; break; case 'c': S[Slen] = 'C'; break; case 'g': S[Slen] = 'G'; break; case 't': S[Slen] = 'T'; break; #else case 'a': S[Slen] = 'a'; break; case 'c': S[Slen] = 'c'; break; case 'g': S[Slen] = 'g'; break; case 't': S[Slen] = 't'; break; #endif case 'A': S[Slen] = 'A'; break; case 'C': S[Slen] = 'C'; break; case 'G': S[Slen] = 'G'; break; case 'T': S[Slen] = 'T'; break; case 'n': S[Slen] = 'N'; break; case 'N': S[Slen] = 'N'; break; default: baseErrors++; S[Slen] = 'N'; break; } Slen++; } // Grab the next line. It should be more sequence, or the next header, or eof. // The last two are stop conditions for the while loop. L[0] = 0; fgets(L, AS_MAX_READLEN+1, F->file()); nLines++; chomp(L); } // Terminate the sequence. S[Slen] = 0; // Report errors. if (baseErrors > 0) { fprintf(errorLog, "read '%s' has "F_U32" invalid base%s. Converted to 'N'.\n", H, baseErrors, (baseErrors > 1) ? "s" : ""); nWARNS++; } if (Slen == 0) { fprintf(errorLog, "read '%s' is empty.\n", H); nWARNS++; } if (Slen != nBases) { fprintf(errorLog, "read '%s' is too long; contains %u bases, but we can only handle %u.\n", H, nBases, AS_MAX_READLEN); nWARNS++; } // Do NOT clear L, it contains the next header. return(nLines); } uint32 loadFASTQ(char *L, char *H, char *S, uint32 &Slen, char *Q, compressedFileReader *F, FILE *errorLog, uint32 &nWARNS) { // We've already read the header. It's in L. strcpy(H, L + 1); // Load sequence. S[0] = 0; Slen = 0; S[AS_MAX_READLEN+1-2] = 0; // If this is ever set, the read is probably longer than we can support. S[AS_MAX_READLEN+1-1] = 0; // This will always be zero; fgets() sets it. Q[AS_MAX_READLEN+1-2] = 0; // This too. Q[AS_MAX_READLEN+1-1] = 0; fgets(S, AS_MAX_READLEN+1, F->file()); chomp(S); // Check for long reads. If found, read the rest of the line, and report an error. The -1 (in // the print) is because fgets() and strlen() will count the newline, which isn't a base. if ((S[AS_MAX_READLEN+1-2] != 0) && (S[AS_MAX_READLEN+1-2] != '\n')) { char *overflow = new char [1048576]; uint32 nBases = AS_MAX_READLEN; do { overflow[1048576-2] = 0; overflow[1048576-1] = 0; fgets(overflow, 1048576, F->file()); nBases += strlen(overflow); } while (overflow[1048576-2] != 0); fprintf(errorLog, "read '%s' is too long; contains %u bases, but we can only handle %u.\n", H, nBases-1, AS_MAX_READLEN); nWARNS++; delete [] overflow; } // Check for and correct invalid bases. uint32 baseErrors = 0; for (uint32 i=0; (Slen < AS_MAX_READLEN) && (S[i] != 0); i++) { switch (S[i]) { #ifdef UPCASE case 'a': S[i] = 'A'; break; case 'c': S[i] = 'C'; break; case 'g': S[i] = 'G'; break; case 't': S[i] = 'T'; break; #else case 'a': break; case 'c': break; case 'g': break; case 't': break; #endif case 'A': break; case 'C': break; case 'G': break; case 'T': break; case 'n': S[i] = 'N'; break; case 'N': break; default: S[i] = 'N'; Q[i] = '!'; // QV=0, ASCII=33 baseErrors++; break; } Slen++; } if (baseErrors > 0) { fprintf(errorLog, "read '%s' has "F_U32" invalid base%s. Converted to 'N'.\n", L, baseErrors, (baseErrors > 1) ? "s" : ""); nWARNS++; } // Load the qv header, and then load the qvs themselves over the header. Q[0] = 0; fgets(Q, AS_MAX_READLEN+1, F->file()); fgets(Q, AS_MAX_READLEN+1, F->file()); chomp(Q); // As with the base, we need to suck in the rest of the longer-than-allowed QV string. But we don't need to report it // or do anything fancy, just advance the file pointer. if ((Q[AS_MAX_READLEN-1] != 0) && (Q[AS_MAX_READLEN-1] != '\n')) { char *overflow = new char [1048576]; do { overflow[1048576-2] = 0; overflow[1048576-1] = 0; fgets(overflow, 1048576, F->file()); } while (overflow[1048576-2] != 0); delete [] overflow; } // Convert from the (assumed to be) Sanger QVs to plain ol' integers. uint32 QVerrors = 0; #ifndef DO_NOT_STORE_QVs for (uint32 i=0; Q[i]; i++) { if (Q[i] < '!') { // QV=0, ASCII=33 Q[i] = '!'; QVerrors++; } if (Q[i] > '!' + 60) { // QV=60, ASCII=93=']' Q[i] = '!' + 60; QVerrors++; } } if (QVerrors > 0) { fprintf(errorLog, "read '%s' has "F_U32" invalid QV%s. Converted to min or max value.\n", L, QVerrors, (QVerrors > 1) ? "s" : ""); nWARNS++; } #else // If we're not using QVs, just reset the first value to -1. This is the sentinel that FASTA sequences set, // causing the encoding later to use a fixed QV for all bases. Q[0] = 0; #endif // Clear the lines, so we can load the next one. L[0] = 0; return(4); // FASTQ always reads exactly four lines } void loadReads(gkStore *gkpStore, gkLibrary *gkpLibrary, uint32 gkpFileID, uint32 minReadLength, FILE *nameMap, FILE *htmlLog, FILE *errorLog, char *fileName, uint32 &nWARNS, uint32 &nLOADED, uint64 &bLOADED, uint32 &nSKIPPED, uint64 &bSKIPPED) { char *L = new char [AS_MAX_READLEN + 1]; // +1. One for the newline, and one for the terminating nul. char *H = new char [AS_MAX_READLEN + 1]; char *S = new char [AS_MAX_READLEN + 1]; char *Q = new char [AS_MAX_READLEN + 1]; uint32 Slen = 0; uint32 Qlen = 0; uint64 lineNumber = 1; fprintf(stderr, "\n"); fprintf(stderr, " Loading reads from '%s'\n", fileName); #if 0 fprintf(htmlLog, "%s\n", gkpFileID, fileName); fprintf(htmlLog, "Parameterspreset=N/A\n"); fprintf(htmlLog, "defaultQV=%u\n", gkpLibrary->gkLibrary_defaultQV()); fprintf(htmlLog, "isNonRandom=%s\n", gkpLibrary->gkLibrary_isNonRandom() ? "true" : "false"); fprintf(htmlLog, "removeDuplicateReads=%s\n", gkpLibrary->gkLibrary_removeDuplicateReads() ? "true" : "false"); fprintf(htmlLog, "finalTrim=%s\n", gkpLibrary->gkLibrary_finalTrim() ? "true" : "false"); fprintf(htmlLog, "removeSpurReads=%s\n", gkpLibrary->gkLibrary_removeSpurReads() ? "true" : "false"); fprintf(htmlLog, "removeChimericReads=%s\n", gkpLibrary->gkLibrary_removeChimericReads() ? "true" : "false"); fprintf(htmlLog, "checkForSubReads=%s\n", gkpLibrary->gkLibrary_checkForSubReads() ? "true" : "false"); #else fprintf(htmlLog, "nam "F_U32" %s\n", gkpFileID, fileName); fprintf(htmlLog, "lib preset=N/A"); fprintf(htmlLog, " defaultQV=%u", gkpLibrary->gkLibrary_defaultQV()); fprintf(htmlLog, " isNonRandom=%s", gkpLibrary->gkLibrary_isNonRandom() ? "true" : "false"); fprintf(htmlLog, " removeDuplicateReads=%s", gkpLibrary->gkLibrary_removeDuplicateReads() ? "true" : "false"); fprintf(htmlLog, " finalTrim=%s", gkpLibrary->gkLibrary_finalTrim() ? "true" : "false"); fprintf(htmlLog, " removeSpurReads=%s", gkpLibrary->gkLibrary_removeSpurReads() ? "true" : "false"); fprintf(htmlLog, " removeChimericReads=%s", gkpLibrary->gkLibrary_removeChimericReads() ? "true" : "false"); fprintf(htmlLog, " checkForSubReads=%s\n", gkpLibrary->gkLibrary_checkForSubReads() ? "true" : "false"); #endif compressedFileReader *F = new compressedFileReader(fileName); uint32 nFASTAlocal = 0; // number of sequences read from disk uint32 nFASTQlocal = 0; uint32 nWARNSlocal = 0; uint32 nLOADEDAlocal = 0; // Sequences actaully loaded into the store uint32 nLOADEDQlocal = 0; uint64 bLOADEDAlocal = 0; uint64 bLOADEDQlocal = 0; uint32 nSKIPPEDAlocal = 0; // Sequences skipped because they are too short uint32 nSKIPPEDQlocal = 0; uint64 bSKIPPEDAlocal = 0; uint64 bSKIPPEDQlocal = 0; fgets(L, AS_MAX_READLEN+1, F->file()); chomp(L); while (!feof(F->file())) { bool isFASTA = false; bool isFASTQ = false; if (L[0] == '>') { lineNumber += loadFASTA(L, H, S, Slen, Q, F, errorLog, nWARNSlocal); isFASTA = true; nFASTAlocal++; } else if (L[0] == '@') { lineNumber += loadFASTQ(L, H, S, Slen, Q, F, errorLog, nWARNSlocal); isFASTQ = true; nFASTQlocal++; } else { fprintf(errorLog, "invalid read header '%.40s%s' in file '%s' at line "F_U64", skipping.\n", L, (strlen(L) > 80) ? "..." : "", fileName, lineNumber); L[0] = 0; nWARNSlocal++; } // If S[0] isn't nul, we loaded a sequence and need to store it. if (Slen < minReadLength) { fprintf(errorLog, "read '%s' of length "F_U32" in file '%s' at line "F_U64" is too short, skipping.\n", H, Slen, fileName, lineNumber); if (isFASTA) { nSKIPPEDAlocal += 1; bSKIPPEDAlocal += Slen; } if (isFASTQ) { nSKIPPEDQlocal += 1; bSKIPPEDQlocal += Slen; } S[0] = 0; Q[0] = 0; } if (S[0] != 0) { gkRead *nr = gkpStore->gkStore_addEmptyRead(gkpLibrary); gkReadData *nd = nr->gkRead_encodeSeqQlt(H, S, Q, gkpLibrary->gkLibrary_defaultQV()); gkpStore->gkStore_stashReadData(nr, nd); delete nd; if (isFASTA) { nLOADEDAlocal += 1; bLOADEDAlocal += Slen; } if (isFASTQ) { nLOADEDQlocal += 1; bLOADEDQlocal += Slen; } fprintf(nameMap, F_U32"\t%s\n", gkpStore->gkStore_getNumReads(), H); } // If L[0] is nul, we need to load the next line. If not, the next line is the header (from // the fasta loader). if (L[0] == 0) { fgets(L, AS_MAX_READLEN+1, F->file()); lineNumber++; chomp(L); } } delete F; delete [] Q; delete [] S; delete [] H; delete [] L; lineNumber--; // The last fgets() returns EOF, but we still count the line. // Write status to the screen fprintf(stderr, " Processed "F_U64" lines.\n", lineNumber); fprintf(stderr, " Loaded "F_U64" bp from:\n", bLOADEDAlocal + bLOADEDQlocal); if (nFASTAlocal > 0) fprintf(stderr, " "F_U32" FASTA format reads ("F_U64" bp).\n", nFASTAlocal, bLOADEDAlocal); if (nFASTQlocal > 0) fprintf(stderr, " "F_U32" FASTQ format reads ("F_U64" bp).\n", nFASTQlocal, bLOADEDQlocal); if (nWARNSlocal > 0) fprintf(stderr, " WARNING: "F_U32" reads issued a warning.\n", nWARNSlocal); if (nSKIPPEDAlocal > 0) fprintf(stderr, " WARNING: "F_U32" reads (%0.4f%%) with "F_U64" bp (%0.4f%%) were too short (< "F_U32"bp) and were ignored.\n", nSKIPPEDAlocal, 100.0 * nSKIPPEDAlocal / (nSKIPPEDAlocal + nLOADEDAlocal), bSKIPPEDAlocal, 100.0 * bSKIPPEDAlocal / (bSKIPPEDAlocal + bLOADEDAlocal), minReadLength); if (nSKIPPEDQlocal > 0) fprintf(stderr, " WARNING: "F_U32" reads (%0.4f%%) with "F_U64" bp (%0.4f%%) were too short (< "F_U32"bp) and were ignored.\n", nSKIPPEDQlocal, 100.0 * nSKIPPEDQlocal / (nSKIPPEDQlocal + nLOADEDQlocal), bSKIPPEDQlocal, 100.0 * bSKIPPEDQlocal / (bSKIPPEDQlocal + bLOADEDQlocal), minReadLength); // Write status to HTML #if 0 fprintf(htmlLog, "FASTA"F_U32" reads ("F_U64" bp)\n", nLOADEDAlocal, bLOADEDAlocal); fprintf(htmlLog, ""F_U32" reads ("F_U64" bp) were short and not loaded\n", nSKIPPEDAlocal, bSKIPPEDAlocal); fprintf(htmlLog, "FASTQ"F_U32" reads ("F_U64" bp)\n", nLOADEDQlocal, bLOADEDQlocal); fprintf(htmlLog, ""F_U32" reads ("F_U64" bp) were short and not loaded\n", nSKIPPEDQlocal, bSKIPPEDQlocal); fprintf(htmlLog, ""F_U32" reads ("F_U64" bp) loaded, "F_U32" reads ("F_U64" bp) skipped, "F_U32" warnings\n", nLOADEDAlocal + nLOADEDQlocal, bLOADEDAlocal + bLOADEDQlocal, nSKIPPEDAlocal + nSKIPPEDQlocal, bSKIPPEDAlocal + bSKIPPEDQlocal, nWARNSlocal); #else fprintf(htmlLog, "dat "F_U32" "F_U64" "F_U32" "F_U64" "F_U32" "F_U64" "F_U32" "F_U64" "F_U32"\n", nLOADEDAlocal, bLOADEDAlocal, nSKIPPEDAlocal, bSKIPPEDAlocal, nLOADEDQlocal, bLOADEDQlocal, nSKIPPEDQlocal, bSKIPPEDQlocal, nWARNSlocal); #endif // Add the just loaded numbers to the global numbers nWARNS += nWARNSlocal; nLOADED += nLOADEDAlocal + nLOADEDQlocal; bLOADED += bLOADEDAlocal + bLOADEDQlocal; nSKIPPED += nSKIPPEDAlocal + nSKIPPEDQlocal; bSKIPPED += bSKIPPEDAlocal + bSKIPPEDQlocal; }; int main(int argc, char **argv) { char *gkpStoreName = NULL; char *outPrefix = NULL; uint32 minReadLength = 0; uint32 firstFileArg = 0; char errorLogName[FILENAME_MAX]; char htmlLogName[FILENAME_MAX]; char nameMapName[FILENAME_MAX]; argc = AS_configure(argc, argv); int arg = 1; int err = 0; while (arg < argc) { if (strcmp(argv[arg], "-o") == 0) { gkpStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-minlength") == 0) { minReadLength = atoi(argv[++arg]); } else if (strcmp(argv[arg], "--") == 0) { firstFileArg = arg++; break; } else if (argv[arg][0] == '-') { fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); err++; } else { firstFileArg = arg; break; } arg++; } if (gkpStoreName == NULL) err++; if (firstFileArg == 0) err++; if (err) { fprintf(stderr, "usage: %s [...] -o gkpStore\n", argv[0]); fprintf(stderr, " -o gkpStore create this gkpStore\n"); fprintf(stderr, " \n"); fprintf(stderr, " -minlength L discard reads shorter than L\n"); fprintf(stderr, " \n"); fprintf(stderr, " \n"); if (gkpStoreName == NULL) fprintf(stderr, "ERROR: no gkpStore (-o) supplied.\n"); if (firstFileArg == 0) fprintf(stderr, "ERROR: no input files supplied.\n"); exit(1); } gkStore *gkpStore = gkStore::gkStore_open(gkpStoreName, gkStore_extend); gkRead *gkpRead = NULL; gkLibrary *gkpLibrary = NULL; uint32 gkpFileID = 0; // Used for HTML output, an ID for each file loaded. uint32 inLineLen = 1024; char inLine[1024] = { 0 }; validSeq['a'] = validSeq['c'] = validSeq['g'] = validSeq['t'] = validSeq['n'] = 1; validSeq['A'] = validSeq['C'] = validSeq['G'] = validSeq['T'] = validSeq['N'] = 1; errno = 0; sprintf(errorLogName, "%s/errorLog", gkpStoreName); FILE *errorLog = fopen(errorLogName, "w"); if (errno) fprintf(stderr, "ERROR: cannot open error file '%s': %s\n", errorLogName, strerror(errno)), exit(1); sprintf(htmlLogName, "%s/load.dat", gkpStoreName); FILE *htmlLog = fopen(htmlLogName, "w"); if (errno) fprintf(stderr, "ERROR: cannot open uid map file '%s': %s\n", htmlLogName, strerror(errno)), exit(1); sprintf(nameMapName, "%s/readNames.txt", gkpStoreName); FILE *nameMap = fopen(nameMapName, "w"); if (errno) fprintf(stderr, "ERROR: cannot open uid map file '%s': %s\n", nameMapName, strerror(errno)), exit(1); uint32 nERROR = 0; // There aren't any errors, we just exit fatally if encountered. uint32 nWARNS = 0; uint32 nLOADED = 0; // Reads loaded uint64 bLOADED = 0; // Bases loaded uint32 nSKIPPED = 0; uint64 bSKIPPED = 0; // Bases not loaded, too short #if 0 fprintf(htmlLog, "\n"); fprintf(htmlLog, "\n"); fprintf(htmlLog, "\n"); fprintf(htmlLog, "gatekeeper load statistics\n"); fprintf(htmlLog, "\n"); fprintf(htmlLog, "\n"); fprintf(htmlLog, "\n"); fprintf(htmlLog, "

Input Files

\n"); fprintf(htmlLog, "\n"); #endif for (; firstFileArg < argc; firstFileArg++) { fprintf(stderr, "\n"); fprintf(stderr, "Starting file '%s'.\n", argv[firstFileArg]); compressedFileReader *inFile = new compressedFileReader(argv[firstFileArg]); char *line = new char [10240]; char *linekv = new char [10240]; KeyAndValue keyval; while (fgets(line, 10240, inFile->file()) != NULL) { chomp(line); strcpy(linekv, line); // keyval.find() modifies the input line, adding a nul byte to split the key and value. keyval.find(linekv); if (keyval.key() == NULL) { // No key, so must be a comment or blank line continue; } if (strcasecmp(keyval.key(), "name") == 0) { gkpLibrary = gkpStore->gkStore_addEmptyLibrary(keyval.value()); continue; } // We'd better have a gkpLibrary defined, if not, the .gkp input file is incorrect. if (gkpLibrary == NULL) { fprintf(stderr, "WARNING: no 'name' tag in gkp input; creating library with name 'DEFAULT'.\n"); gkpLibrary = gkpStore->gkStore_addEmptyLibrary(keyval.value()); nWARNS++; } if (strcasecmp(keyval.key(), "preset") == 0) { gkpLibrary->gkLibrary_parsePreset(keyval.value()); } else if (strcasecmp(keyval.key(), "qv") == 0) { gkpLibrary->gkLibrary_setDefaultQV(keyval.value_double()); } else if (strcasecmp(keyval.key(), "isNonRandom") == 0) { gkpLibrary->gkLibrary_setIsNonRandom(keyval.value_bool()); } else if (strcasecmp(keyval.key(), "readType") == 0) { gkpLibrary->gkLibrary_setReadType(keyval.value()); } else if (strcasecmp(keyval.key(), "removeDuplicateReads") == 0) { gkpLibrary->gkLibrary_setRemoveDuplicateReads(keyval.value_bool()); } else if (strcasecmp(keyval.key(), "finalTrim") == 0) { gkpLibrary->gkLibrary_setFinalTrim(keyval.value()); } else if (strcasecmp(keyval.key(), "removeSpurReads") == 0) { gkpLibrary->gkLibrary_setRemoveSpurReads(keyval.value_bool()); } else if (strcasecmp(keyval.key(), "removeChimericReads") == 0) { gkpLibrary->gkLibrary_setRemoveChimericReads(keyval.value_bool()); } else if (strcasecmp(keyval.key(), "checkForSubReads") == 0) { gkpLibrary->gkLibrary_setCheckForSubReads(keyval.value_bool()); } else if (AS_UTL_fileExists(line, false, false)) { loadReads(gkpStore, gkpLibrary, gkpFileID++, minReadLength, nameMap, htmlLog, errorLog, line, nWARNS, nLOADED, bLOADED, nSKIPPED, bSKIPPED); } else { fprintf(stderr, "ERROR: option '%s' not recognized, and not a file of reads.\n", line); exit(1); } } delete inFile; delete [] line; delete [] linekv; } #if 0 fprintf(htmlLog, "
\n"); #endif gkpStore->gkStore_close(); fclose(nameMap); fclose(errorLog); fprintf(stderr, "\n"); fprintf(stderr, "Finished with:\n"); fprintf(stderr, " "F_U32" warnings (bad base or qv, too short, too long)\n", nWARNS); fprintf(stderr, "\n"); #if 0 fprintf(stderr, "Read from inputs:\n"); fprintf(stderr, " "F_U64" bp.\n", bLOADED); fprintf(stderr, " "F_U32" reads.\n", nLOADED); fprintf(stderr, "\n"); #endif fprintf(stderr, "Loaded into store:\n"); fprintf(stderr, " "F_U64" bp.\n", bLOADED); fprintf(stderr, " "F_U32" reads.\n", nLOADED); fprintf(stderr, "\n"); fprintf(stderr, "Skipped (too short):\n"); fprintf(stderr, " "F_U64" bp (%.4f%%).\n", bSKIPPED, 100.0 * bSKIPPED / (bSKIPPED + bLOADED)); fprintf(stderr, " "F_U32" reads (%.4f%%).\n", nSKIPPED, 100.0 * nSKIPPED / (nSKIPPED + nLOADED)); fprintf(stderr, "\n"); fprintf(stderr, "\n"); #if 0 fprintf(htmlLog, "\n"); fprintf(htmlLog, "

Final Store

\n"); fprintf(htmlLog, "\n"); fprintf(htmlLog, "\n", gkpStoreName); fprintf(htmlLog, "\n", nLOADED, bLOADED); fprintf(htmlLog, "\n", nSKIPPED, bSKIPPED); fprintf(htmlLog, "\n", nWARNS); fprintf(htmlLog, "
%s
readsLoaded"F_U32" reads ("F_U64" bp)
readsSkipped"F_U32" reads ("F_U64" bp) (read was too short)
warnings"F_U32" warnings (invalid base or quality value)
\n"); fprintf(htmlLog, "\n"); fprintf(htmlLog, "\n"); fprintf(htmlLog, "\n"); fprintf(htmlLog, "\n"); fprintf(htmlLog, "\n"); #else fprintf(htmlLog, "sum "F_U32" "F_U64" "F_U32" "F_U64" "F_U32"\n", nLOADED, bLOADED, nSKIPPED, bSKIPPED, nWARNS); #endif fclose(htmlLog); if (nERROR > 0) fprintf(stderr, "gatekeeperCreate did NOT finish successfully; too many errors.\n"); if (bSKIPPED > 0.25 * (bSKIPPED + bLOADED)) fprintf(stderr, "gatekeeperCreate did NOT finish successfully; too many bases skipped. Check your reads.\n"); if (nWARNS > 0.25 * (nLOADED)) fprintf(stderr, "gatekeeperCreate did NOT finish successfully; too many warnings. Check your reads.\n"); if (nSKIPPED > 0.50 * (nLOADED)) fprintf(stderr, "gatekeeperCreate did NOT finish successfully; too many short reads. Check your reads!\n"); if ((nERROR > 0) || (bSKIPPED > 0.25 * (bSKIPPED + bLOADED)) || (nWARNS > 0.25 * (nSKIPPED + nLOADED)) || (nSKIPPED > 0.50 * (nSKIPPED + nLOADED))) exit(1); fprintf(stderr, "gatekeeperCreate finished successfully.\n"); exit(0); } canu-1.3/src/stores/gatekeeperCreate.mk000066400000000000000000000007411272335422300202070ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := gatekeeperCreate SOURCES := gatekeeperCreate.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/gatekeeperDumpFASTQ.C000066400000000000000000000264201272335422300202650ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_GKP/gkpStoreDumpFASTQ.C * * Modifications by: * * Brian P. Walenz from 2012-FEB-06 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-21 to 2015-JUN-03 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-23 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "AS_UTL_decodeRange.H" #include "AS_UTL_fileIO.H" #include "AS_UTL_fasta.H" #include "clearRangeFile.H" // Write sequence in multiple formats. This used to write to four fastq files, the .1, .2, .paired and .unmated. // It's left around for future expansion to .fastq and .bax.h5. // class libOutput { public: libOutput(char const *outPrefix, char const *outSuffix, char const *libName = NULL) { strcpy(_p, outPrefix); if (outSuffix[0]) sprintf(_s, ".%s", outSuffix); else _s[0] = 0; if (libName) strcpy(_n, libName); else _n[0] = 0; _WRITER = NULL; _FASTA = NULL; _FASTQ = NULL; }; ~libOutput() { if (_WRITER) delete _WRITER; }; FILE *getFASTQ(void) { if (_FASTQ) return(_FASTQ); return(openFASTQ()); }; FILE *openFASTQ(void) { char N[FILENAME_MAX]; if (_n[0]) sprintf(N, "%s.%s.fastq%s", _p, _n, _s); else sprintf(N, "%s.fastq%s", _p, _s); if ((_p[0] == '-') && (_p[1] == 0)) { sprintf(N, "(stdout)"); _FASTQ = stdout; } else { _WRITER = new compressedFileWriter(N); _FASTQ = _WRITER->file(); } return(_FASTQ); }; FILE *getFASTA(void) { if (_FASTA) return(_FASTA); return(openFASTA()); }; FILE *openFASTA(void) { char N[FILENAME_MAX]; if (_n[0]) sprintf(N, "%s.%s.fasta%s", _p, _n, _s); else sprintf(N, "%s.fasta%s", _p, _s); if ((_p[0] == '-') && (_p[1] == 0)) { sprintf(N, "(stdout)"); _FASTA = stdout; } else { _WRITER = new compressedFileWriter(N); _FASTA = _WRITER->file(); } return(_FASTA); }; private: char _p[FILENAME_MAX]; char _s[FILENAME_MAX]; char _n[FILENAME_MAX]; compressedFileWriter *_WRITER; FILE *_FASTA; FILE *_FASTQ; }; char * scanPrefix(char *prefix) { int32 len = strlen(prefix); if ((len > 3) && (strcasecmp(prefix + len - 3, ".gz") == 0)) { prefix[len-3] = 0; return(prefix + len - 2); } if ((len > 4) && (strcasecmp(prefix + len - 4, ".bz2") == 0)) { prefix[len-4] = 0; return(prefix + len - 3); } if ((len > 3) && (strcasecmp(prefix + len - 3, ".xz") == 0)) { prefix[len-3] = 0; return(prefix + len - 2); } return(prefix + len); } int main(int argc, char **argv) { char *gkpStoreName = NULL; char *outPrefix = NULL; char *outSuffix = NULL; char *clrName = NULL; uint32 libToDump = 0; uint32 bgnID = 1; uint32 endID = UINT32_MAX; bool dumpAllReads = false; bool dumpAllBases = false; bool dumpOnlyDeleted = false; bool dumpFASTQ = true; bool dumpFASTA = false; bool withLibName = true; argc = AS_configure(argc, argv); int arg = 1; int err = 0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-o") == 0) { outPrefix = argv[++arg]; outSuffix = scanPrefix(outPrefix); } else if (strcmp(argv[arg], "-c") == 0) { clrName = argv[++arg]; } else if (strcmp(argv[arg], "-l") == 0) { libToDump = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-b") == 0) { // DEPRECATED! bgnID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-e") == 0) { // DEPRECATED! endID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-r") == 0) { AS_UTL_decodeRange(argv[++arg], bgnID, endID); } else if (strcmp(argv[arg], "-allreads") == 0) { dumpAllReads = true; } else if (strcmp(argv[arg], "-allbases") == 0) { dumpAllBases = true; } else if (strcmp(argv[arg], "-onlydeleted") == 0) { dumpOnlyDeleted = true; dumpAllReads = true; // Otherwise we won't report the deleted reads! } else if (strcmp(argv[arg], "-fastq") == 0) { dumpFASTQ = true; dumpFASTA = false; } else if (strcmp(argv[arg], "-fasta") == 0) { dumpFASTQ = false; dumpFASTA = true; } else if (strcmp(argv[arg], "-nolibname") == 0) { withLibName = false; } else { err++; fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); } arg++; } if (gkpStoreName == NULL) err++; if (outPrefix == NULL) err++; if (err) { fprintf(stderr, "usage: %s [...] -o fastq-prefix -g gkpStore\n", argv[0]); fprintf(stderr, " -G gkpStore\n"); fprintf(stderr, " -o fastq-prefix write files fastq-prefix.(libname).fastq, ...\n"); fprintf(stderr, " if fastq-prefix is '-', all sequences output to stdout\n"); fprintf(stderr, " if fastq-prefix ends in .gz, .bz2 or .xz, output is compressed\n"); fprintf(stderr, "\n"); fprintf(stderr, " -l libToDump output only read in library number libToDump (NOT IMPLEMENTED)\n"); fprintf(stderr, " -r id[-id] output only the single read 'id', or the specified range of ids\n"); fprintf(stderr, "\n"); fprintf(stderr, " -c clearFile clear range file from OBT modules\n"); fprintf(stderr, " -allreads if a clear range file, lower case mask the deleted reads\n"); fprintf(stderr, " -allbases if a clear range file, lower case mask the non-clear bases\n"); fprintf(stderr, " -onlydeleted if a clear range file, only output deleted reads (the entire read)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -fastq output is FASTQ format (with extension .fastq, default)\n"); fprintf(stderr, " -fasta output is FASTA format (with extension .fasta)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -nolibname don't include the library name in the output file name\n"); fprintf(stderr, "\n"); if (gkpStoreName == NULL) fprintf(stderr, "ERROR: no gkpStore (-G) supplied.\n"); if (outPrefix == NULL) fprintf(stderr, "ERROR: no output prefix (-o) supplied.\n"); exit(1); } gkStore *gkpStore = gkStore::gkStore_open(gkpStoreName); uint32 numReads = gkpStore->gkStore_getNumReads(); uint32 numLibs = gkpStore->gkStore_getNumLibraries(); clearRangeFile *clrRange = (clrName == NULL) ? NULL : new clearRangeFile(clrName, gkpStore); if (bgnID < 1) bgnID = 1; if (numReads < endID) endID = numReads; if (endID < bgnID) fprintf(stderr, "No reads to dump; reversed ranges make no sense: bgn="F_U32" end="F_U32"??\n", bgnID, endID); fprintf(stderr, "Dumping reads from %u to %u (inclusive).\n", bgnID, endID); libOutput **out = new libOutput * [numLibs + 1]; // Allocate outputs. If withLibName == false, all reads will artificially be in lib zero, the // other files won't ever be created. Otherwise, the zeroth file won't ever be created. out[0] = new libOutput(outPrefix, outSuffix, NULL); for (uint32 i=1; i<=numLibs; i++) out[i] = new libOutput(outPrefix, outSuffix, gkpStore->gkStore_getLibrary(i)->gkLibrary_libraryName()); // Grab a new readData, and iterate through reads to dump. gkReadData *readData = new gkReadData; for (uint32 rid=bgnID; rid<=endID; rid++) { gkRead *read = gkpStore->gkStore_getRead(rid); uint32 libID = (withLibName == false) ? 0 : read->gkRead_libraryID(); uint32 flen = read->gkRead_sequenceLength(); uint32 lclr = 0; uint32 rclr = flen; bool ignore = false; //fprintf(stderr, "READ %u claims id %u length %u in lib %u\n", rid, read->gkRead_readID(), read->gkRead_sequenceLength(), libID); // If a clear range file is supplied, grab the clear range. If it hasn't been set, the default // is the entire read. if (clrRange) { lclr = clrRange->bgn(rid); rclr = clrRange->end(rid); ignore = clrRange->isDeleted(rid); } // Abort if we're not dumping anything from this read // - not in a library we care about // - deleted, and not dumping all reads // - not deleted, but only reporting deleted reads if (((libToDump != 0) && (libID == libToDump)) || ((dumpAllReads == false) && (ignore == true)) || ((dumpOnlyDeleted == true) && (ignore == false))) continue; // And if we're told to ignore the read, and here, then the read was deleted and we're printing // all reads. Reset the clear range to the whole read, the clear range is invalid. if (ignore) { lclr = 0; rclr = read->gkRead_sequenceLength(); } // Grab the sequence and quality. gkpStore->gkStore_loadReadData(read, readData); char *seq = readData->gkReadData_getSequence(); char *qlt = readData->gkReadData_getQualities(); uint32 clen = rclr - lclr; // Soft mask not-clear bases if (dumpAllBases == true) { for (uint32 i=0; i= 'A') ? 'a' - 'A' : 0; for (uint32 i=lclr; i= 'A') ? 0 : 'A' - 'a'; for (uint32 i=rclr; flen; i++) seq[i] += (seq[i] >= 'A') ? 'a' - 'A' : 0; lclr = 0; rclr = flen; } // Chop off the ends we're not printing. seq += lclr; qlt += lclr; seq[clen] = 0; qlt[clen] = 0; // Print the read. if (dumpFASTA) AS_UTL_writeFastA(out[libID]->getFASTA(), seq, clen, 100, ">"F_U32" clr="F_U32","F_U32"\n", rid, lclr, rclr); if (dumpFASTQ) AS_UTL_writeFastQ(out[libID]->getFASTQ(), seq, clen, qlt, clen, "@"F_U32" clr="F_U32","F_U32"\n", rid, lclr, rclr); } delete clrRange; delete readData; for (uint32 i=0; i<=numLibs; i++) delete out[i]; delete [] out; gkpStore->gkStore_close(); exit(0); } canu-1.3/src/stores/gatekeeperDumpFASTQ.mk000066400000000000000000000010111272335422300204770ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := gatekeeperDumpFASTQ SOURCES := gatekeeperDumpFASTQ.C SRC_INCDIRS := .. ../stores ../AS_UTL ../overlapBasedTrimming TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/gatekeeperDumpMetaData.C000066400000000000000000000177351272335422300211000ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-MAR-18 to 2015-SEP-21 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-24 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" void dumpLibs(gkStore *gkp, uint32 bgnID, uint32 endID) { //fprintf(stderr, "Dumping libraries from %u to %u (inclusive).\n", bgnID, endID); fprintf(stdout, "libID\tnonRandom\treadType\tcorrectBases\tfinalTrim\tremoveDupe\tremoveSpur\tremoveChimer\tcheckSubRead\tdefaultQV\tlibName\n"); for (uint32 lid=bgnID; lid<=endID; lid++) { gkLibrary *library = gkp->gkStore_getLibrary(lid); fprintf(stdout, F_U32"\t"F_U32"\t%s\t%s\t%s\t"F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\t"F_U32"\t%s\n", library->gkLibrary_libraryID(), library->gkLibrary_isNonRandom(), library->gkLibrary_readTypeString(), library->gkLibrary_readCorrectionString(), library->gkLibrary_finalTrimString(), library->gkLibrary_removeDuplicateReads(), library->gkLibrary_removeSpurReads(), library->gkLibrary_removeChimericReads(), library->gkLibrary_checkForSubReads(), library->gkLibrary_defaultQV(), library->gkLibrary_libraryName()); } } void dumpReads(gkStore *gkp, uint32 bgnID, uint32 endID, bool fullDump) { //fprintf(stderr, "Dumping reads from %u to %u (inclusive).\n", bgnID, endID); for (uint32 rid=bgnID; rid<=endID; rid++) { gkRead *read = gkp->gkStore_getReadInPartition(rid); if (read == NULL) continue; if (fullDump == false) fprintf(stdout, F_U32"\t"F_U32"\t"F_U32"\n", read->gkRead_readID(), read->gkRead_libraryID(), read->gkRead_sequenceLength()); else fprintf(stdout, F_U32"\t"F_U32"\t"F_U32"\t"F_U64"\t"F_U64"\n", read->gkRead_readID(), read->gkRead_libraryID(), read->gkRead_sequenceLength(), read->gkRead_mPtr(), read->gkRead_pID()); } } class readStats { public: readStats() { _nBases = 0; _minBases = UINT32_MAX; _maxBases = 0; }; ~readStats() { }; void addRead(gkRead *read) { uint32 l = read->gkRead_sequenceLength(); _readLengths.push_back(l); _nBases += l; if (l < _minBases) _minBases = l; if (_maxBases < l) _maxBases = l; }; uint32 numberOfReads(void) { return(_readLengths.size()); }; uint64 numberOfBases(void) { return(_nBases); }; uint64 minBases(void) { return(_minBases); }; uint64 maxBases(void) { return(_maxBases); }; private: vector _readLengths; uint64 _nBases; uint32 _minBases; uint32 _maxBases; }; void dumpStats(gkStore *gkp, uint32 bgnID, uint32 endID) { //fprintf(stderr, "Dumping read statistics from %u to %u (inclusive).\n", bgnID, endID); readStats *rs = new readStats [gkp->gkStore_getNumLibraries() + 1]; for (uint32 rid=bgnID; rid<=endID; rid++) { gkRead *read = gkp->gkStore_getReadInPartition(rid); if (read == NULL) continue; uint32 l = read->gkRead_libraryID(); rs[0].addRead(read); rs[l].addRead(read); } // Stats per library (this mode when -libs -stats is selected?) and global. // Stats include: // number of reads // total bases // min, mean, stddev, max base per read // length histogram plot for (uint32 l=0; lgkStore_getNumLibraries() + 1; l++) { fprintf(stdout, "library "F_U32" reads "F_U32" bases: total "F_U64" ave "F_U64" min "F_U64" max "F_U64"\n", l, rs[l].numberOfReads(), rs[l].numberOfBases(), rs[l].numberOfBases() / rs[l].numberOfReads(), rs[l].minBases(), rs[l].maxBases()); } } int main(int argc, char **argv) { char *gkpStoreName = NULL; uint32 gkpStorePart = UINT32_MAX; bool wantLibs = false; bool wantReads = true; bool wantStats = false; // Useful only for reads bool fullDump = true; uint32 bgnID = 1; uint32 endID = UINT32_MAX; argc = AS_configure(argc, argv); int arg = 1; int err = 0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpStoreName = argv[++arg]; if ((arg+1 < argc) && (argv[arg+1][0] != '-')) gkpStorePart = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-libs") == 0) { wantLibs = true; wantReads = false; wantStats = false; } else if (strcmp(argv[arg], "-reads") == 0) { wantLibs = false; wantReads = true; wantStats = false; } else if (strcmp(argv[arg], "-full") == 0) { fullDump = true; } else if (strcmp(argv[arg], "-stats") == 0) { wantLibs = false; wantReads = false; wantStats = true; } else if (strcmp(argv[arg], "-b") == 0) { bgnID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-e") == 0) { endID = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-r") == 0) { bgnID = atoi(argv[++arg]); endID = bgnID; } else { err++; fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); } arg++; } if (gkpStoreName == NULL) err++; if (err) { fprintf(stderr, "usage: %s -G gkpStore [p] [...]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " -G gkpStore [p] dump reads from 'gkpStore', restricted to\n"); fprintf(stderr, " partition 'p', if supplied.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -libs dump information about libraries\n"); fprintf(stderr, " -reads [-full] dump information about reads\n"); fprintf(stderr, " (-full also dumps some storage metadata)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -stats dump summary statistics on reads\n"); fprintf(stderr, "\n"); fprintf(stderr, " -b id output starting at read/library 'id'\n"); fprintf(stderr, " -e id output stopping after read/library 'id'\n"); fprintf(stderr, "\n"); fprintf(stderr, " -r id output only the single read 'id'\n"); fprintf(stderr, "\n"); if (gkpStoreName == NULL) fprintf(stderr, "ERROR: no gkpStore (-G) supplied.\n"); exit(1); } gkStore *gkpStore = gkStore::gkStore_open(gkpStoreName, gkStore_readOnly, gkpStorePart); uint32 numReads = gkpStore->gkStore_getNumReads(); uint32 numLibs = gkpStore->gkStore_getNumLibraries(); if (wantLibs) { if (bgnID < 1) bgnID = 1; if (numLibs < endID) endID = numLibs; } else { if (bgnID < 1) bgnID = 1; if (numReads < endID) endID = numReads; } if (endID < bgnID) fprintf(stderr, "No objects to dump; reversed ranges make no sense: bgn="F_U32" end="F_U32"??\n", bgnID, endID); if (wantLibs) dumpLibs(gkpStore, bgnID, endID); if (wantReads) dumpReads(gkpStore, bgnID, endID, fullDump); if (wantStats) dumpStats(gkpStore, bgnID, endID); gkpStore->gkStore_close(); exit(0); } canu-1.3/src/stores/gatekeeperDumpMetaData.mk000066400000000000000000000007551272335422300213170ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := gatekeeperDumpMetaData SOURCES := gatekeeperDumpMetaData.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/gatekeeperPartition.C000066400000000000000000000065411272335422300205340ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2014-DEC-23 to 2015-MAR-17 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "AS_UTL_fileIO.H" int main(int argc, char **argv) { char *gkpStoreName = NULL; char *partitionFile = NULL; argc = AS_configure(argc, argv); int arg = 1; int err = 0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpStoreName = argv[++arg]; } else if (strcmp(argv[arg], "-P") == 0) { partitionFile = argv[++arg]; } else { err++; fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); } arg++; } if (gkpStoreName == NULL) err++; if (partitionFile == NULL) err++; if (err) { fprintf(stderr, "usage: %s -G gkpStore -P partitionMapFile\n", argv[0]); fprintf(stderr, " -G gkpStore path to gatekeeper store\n"); fprintf(stderr, " -P partFile file mapping read ID to partiton\n"); fprintf(stderr, " format: 'partition readID'\n"); fprintf(stderr, " \n"); if (gkpStoreName == NULL) fprintf(stderr, "ERROR: no gkpStore (-G) supplied.\n"); if (partitionFile == NULL) fprintf(stderr, "ERROR: no partition input (-P) supplied.\n"); exit(1); } // Open a READ ONLY store. This prevents us from mucking with the non-partitioned reads // (like, by changing the read ID or pointer to the blob). We don't need it opened // for writing anyway. gkStore *gkpStore = gkStore::gkStore_open(gkpStoreName, gkStore_readOnly); uint32 numReads = gkpStore->gkStore_getNumReads(); uint32 *partition = new uint32 [numReads + 1]; // Set all partitions to invalid. for (uint32 i=0; i<=numReads; i++) partition[i] = UINT32_MAX; // Read the partition file errno = 0; FILE *F = fopen(partitionFile, "r"); if (errno) fprintf(stderr, "GKP Error: Build_Partition()-- failed to open '%s': %s\n", partitionFile, strerror(errno)), exit(1); while (!feof(F)) { uint32 i, p; if (2 == fscanf(F, " "F_U32" "F_U32" ", &p, &i)) { assert(i <= numReads); assert(partition[i] == UINT32_MAX); partition[i] = p; } } fclose(F); // Dump the partition data to the store, let it build partitions. gkpStore->gkStore_buildPartitions(partition); // That's all folks. delete [] partition; gkpStore->gkStore_close(); exit(0); } canu-1.3/src/stores/gatekeeperPartition.mk000066400000000000000000000007471272335422300207630ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := gatekeeperPartition SOURCES := gatekeeperPartition.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/gkStore.C000066400000000000000000001171251272335422300161450ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2014-NOV-26 to 2015-AUG-10 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-09 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2015-DEC-09 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "gkStore.H" #include "AS_UTL_fileIO.H" #include "AS_UTL_alloc.H" gkStore *gkStore::_instance = NULL; uint32 gkStore::_instanceCount = 0; bool gkRead::gkRead_loadData(gkReadData *readData, void *blobs) { readData->_read = this; // The resize will only increase the space. if the new is less than the max, it returns immediately. resizeArrayPair(readData->_seq, readData->_qlt, readData->_seqAlloc, readData->_seqAlloc, (uint32)_seqLen+1, resizeArray_doNothing); // Where, or where!, is the data? uint64 offset = _mPtr; // One might be tempted to set the readData blob to point to the blob data in the mmap, // but doing so will cause it to be written out again. readData->_blobLen = 0; readData->_blobMax = 0; readData->_blob = NULL; // Instead, we'll use someting horribly similar. uint8 *blob = ((uint8 *)blobs) + offset; char chunk[5]; if ((blob[0] != 'B') && (blob[1] != 'L') && (blob[2] != 'O') && (blob[3] != 'B')) fprintf(stderr, "Index error in read "F_U32" %c mPtr "F_U64" pID "F_U64" expected BLOB, got %c%c%c%c\n", gkRead_readID(), '?', //(_numberOfPartitions == 0) ? 'm' : 'p', _mPtr, _pID, blob[0], blob[1], blob[2], blob[3]); assert(blob[0] == 'B'); assert(blob[1] == 'L'); assert(blob[2] == 'O'); assert(blob[3] == 'B'); uint32 blobLen = *((uint32 *)blob + 1); //fprintf(stderr, "BLOB len %u\n", blobLen); blob += 8; while ((blob[0] != 'S') || (blob[1] != 'T') || (blob[2] != 'O') || (blob[3] != 'P')) { chunk[0] = blob[0]; chunk[1] = blob[1]; chunk[2] = blob[2]; chunk[3] = blob[3]; chunk[4] = 0; uint32 chunkLen = *((uint32 *)blob + 1); #if 1 // Fix for stores built between 9/3/15-9/7/15 when QVs were changed to uniform value but nothing was stored on disk // Loading from these stores lead to random uninitialized QVs // set all QVs to a default value // Should be unnecessary after Dec 8, 2015 and can be removed for (uint32 ii=0; ii<_seqLen; ii++) readData->_qlt[ii] = 20; #endif //fprintf(stderr, "%s len %u\n", chunk, chunkLen); if (strncmp(chunk, "VERS", 4) == 0) { } else if (strncmp(chunk, "QSEQ", 4) == 0) { //fprintf(stderr, "QSEQ not supported.\n"); } else if (strncmp(chunk, "USEQ", 4) == 0) { assert(_seqLen <= chunkLen); assert(_seqLen <= readData->_seqAlloc); memcpy(readData->_seq, blob + 8, _seqLen); readData->_seq[_seqLen] = 0; } else if (strncmp(chunk, "UQLT", 4) == 0) { assert(_seqLen <= chunkLen); assert(_seqLen <= readData->_seqAlloc); memcpy(readData->_qlt, blob + 8, _seqLen); readData->_qlt[_seqLen] = 0; #if 1 // Fix for older gkpStore that encoded QV's with offset '0'. Wasn't RIFF format supposed to solve problems like this? // Old encoding is ASCII was from '0' = 48 to 'l' = 108. // New encoding is integer from 0 to 60. // So, if we see: // 0-47, we're new format. // 48-60 we're either (but STRONGLY likely to be old). // 61-108 we're old format. // Most reads are well below qv=40, so this will be easy. bool isOld = false; for (uint32 ii=0; ii<_seqLen; ii++) { if (readData->_qlt[ii] < 48) { isOld = false; break; } else if (readData->_qlt[ii] < 61) { isOld = true; } else { isOld = true; break; } } if (isOld) for (uint32 ii=0; ii<_seqLen; ii++) readData->_qlt[ii] -= '0'; #endif } else if (strncmp(chunk, "2SEQ", 4) == 0) { gkRead_decode2bit(blob + 8, chunkLen, readData->_seq, _seqLen); } else if (strncmp(chunk, "3SEQ", 4) == 0) { gkRead_decode3bit(blob + 8, chunkLen, readData->_seq, _seqLen); } else if (strncmp(chunk, "4QLT", 4) == 0) { gkRead_decode4bit(blob + 8, chunkLen, readData->_qlt, _seqLen); } else if (strncmp(chunk, "5QLT", 4) == 0) { gkRead_decode5bit(blob + 8, chunkLen, readData->_qlt, _seqLen); } else if (strncmp(chunk, "QVAL", 4) == 0) { uint32 qval = *((uint32 *)blob + 2); for (uint32 ii=0; ii<_seqLen; ii++) readData->_qlt[ii] = qval; } else { fprintf(stderr, "gkRead::gkRead_loadData()-- unknown chunk type '%s' skipped\n", chunk); } blob += 4 + 4 + chunkLen; } return(true); }; // Dump a block of encoded data to disk, then update the gkRead to point to it. // void gkStore::gkStore_stashReadData(gkRead *read, gkReadData *data) { assert(_blobsFile != NULL); read->_mPtr = AS_UTL_ftell(_blobsFile); read->_pID = _partitionID; // 0 if not partitioned //fprintf(stderr, "STASH read %u at position "F_SIZE_T"\n", read->gkRead_readID(), AS_UTL_ftell(_blobsFile)); AS_UTL_safeWrite(_blobsFile, data->_blob, "gkStore_stashReadData::blob", sizeof(char), data->_blobLen); } // Load read metadata and data from a stream. // void gkStore::gkStore_loadReadFromStream(FILE *S, gkRead *read, gkReadData *readData) { char tag[5]; uint32 size; // Mark this as a read. Needed for tgTig::loadFromStreamOrLayout(), and loading this stuff in // utgcns. AS_UTL_safeRead(S, tag, "gkStore::gkStore_loadReadFromStream::tag", sizeof(char), 4); if (strncmp(tag, "READ", 4) != 0) fprintf(stderr, "Failed to load gkRead, got tag '%c%c%c%c' (0x%02x 0x%02x 0x%02x 0x%02x), expected 'READ'.\n", tag[0], tag[1], tag[2], tag[3], tag[0], tag[1], tag[2], tag[3]), exit(1); // Load the read metadata AS_UTL_safeRead(S, read, "gkStore::gkStore_loadReadFromStream::read", sizeof(gkRead), 1); // With some pain, we read the BLOB and its length, then allocate space for the blob // and finsh reading it. AS_UTL_safeRead(S, tag, "gkStore::gkStore_loadReadFromStream::blob", sizeof(int8), 4); AS_UTL_safeRead(S, &size, "gkStore::gkStore_loadReadFromStream::size", sizeof(uint32), 1); uint8 *blob = new uint8 [8 + size]; memcpy(blob, tag, sizeof(uint8) * 4); memcpy(blob+4, &size, sizeof(uint32) * 1); AS_UTL_safeRead(S, blob+8, "gkStore::gkStore_loadReadFromStream::blob", sizeof(char), size); // Unpack the blob into a readData read->_mPtr = 0; read->gkRead_loadData(readData, blob); // And, that's it! Sweet! } // Dump the read metadata and read data to a stream. // void gkStore::gkStore_saveReadToStream(FILE *S, uint32 id) { // Mark this as a read. Needed for tgTig::loadFromStreamOrLayout(), and loading this stuff in // utgcns. fprintf(S, "READ"); // Dump the read metadata gkRead *read = gkStore_getRead(id); AS_UTL_safeWrite(S, read, "gkStore::gkStore_saveReadToStream::read", sizeof(gkRead), 1); // Figure out where the blob actually is, and make sure that it really is a blob uint8 *blob = (uint8 *)_blobs + read->_mPtr; uint32 blobLen = 8 + *((uint32 *)blob + 1); assert(blob[0] == 'B'); assert(blob[1] == 'L'); assert(blob[2] == 'O'); assert(blob[3] == 'B'); // Write the blob to the stream AS_UTL_safeWrite(S, blob, "gkStore::gkStore_saveReadToStream::blob", sizeof(char), blobLen); } // Store the 'len' bytes of data in 'dat' into the class-managed _blob data block. // Ensures that the _blob block is appropriately padded to maintain 32-bit alignment. // void gkReadData::gkReadData_encodeBlobChunk(char const *tag, uint32 len, void *dat) { // Allocate an initial blob if we don't have one if (_blobMax == 0) { _blobLen = 0; _blobMax = 1048576; _blob = new uint8 [_blobMax]; } // Or make it bigger while (_blobMax + 8 + len < _blobMax) { _blobMax *= 2; uint8 *b = new uint8 [_blobMax]; memcpy(b, _blob, sizeof(uint8) * _blobLen); delete [] _blob; _blob = b; } // Figure out how much padding we need to add uint32 pad = 4 - (len % 4); if (pad == 4) pad = 0; // Copy in the chunk id and padded length len += pad; memcpy(_blob + _blobLen, tag, sizeof(char) * 4); _blobLen += sizeof(char) * 4; memcpy(_blob + _blobLen, &len, sizeof(uint32)); _blobLen += sizeof(uint32); len -= pad; // Then the unpadded data and any padding. memcpy(_blob + _blobLen, dat, sizeof(uint8) * len); _blobLen += sizeof(uint8) * len; if (pad > 2) _blob[_blobLen++] = 0; if (pad > 1) _blob[_blobLen++] = 0; if (pad > 0) _blob[_blobLen++] = 0; // Finally, update the total blob length. _blobLen -= 8; memcpy(_blob + 4, &_blobLen, sizeof(uint32)); _blobLen += 8; } gkReadData * gkRead::gkRead_encodeSeqQlt(char *H, char *S, char *Q, uint32 qv) { gkReadData *rd = new gkReadData; uint32 RID = _readID; // Debugging // If there is a QV string, ensure that the lengths are the same. If not, trim or pad the QVs. // Then, convert the expected Sanger-encoded QV's (base='!') to be just integers. uint32 Slen = _seqLen = strlen(S); uint32 Qlen = 0; if (Q[0] != 0) { Qlen = strlen(Q); if (Slen < Qlen) { fprintf(stderr, "-- WARNING: read '%s' sequence length %u != quality length %u; quality bases truncated.\n", H, Slen, Qlen); Q[Slen] = 0; } if (Slen > Qlen) { fprintf(stderr, "-- WARNING: read '%s' sequence length %u != quality length %u; quality bases padded.\n", H, Slen, Qlen); for (uint32 ii=Qlen; iigkReadData_encodeBlobChunk("BLOB", 0, NULL); rd->gkReadData_encodeBlobChunk("VERS", 4, &blobVers); if (seq2Len > 0) rd->gkReadData_encodeBlobChunk("2SEQ", seq2Len, seq); // Two-bit encoded sequence (ACGT only) else if (seq3Len > 0) rd->gkReadData_encodeBlobChunk("3SEQ", seq3Len, seq); // Three-bit encoded sequence (ACGTN) else rd->gkReadData_encodeBlobChunk("USEQ", Slen, S); // Unencoded sequence if (qlt4Len > 0) rd->gkReadData_encodeBlobChunk("4QLT", qlt4Len, qlt); // Four-bit (0-15) encoded QVs else if (qlt5Len > 0) rd->gkReadData_encodeBlobChunk("5QLT", qlt5Len, qlt); // Five-bit (0-32) encoded QVs else if (Q[0] == 0) rd->gkReadData_encodeBlobChunk("QVAL", 4, &qv); // Constant QV for every base else rd->gkReadData_encodeBlobChunk("UQLT", Qlen, Q); // Unencoded quality rd->gkReadData_encodeBlobChunk("STOP", 0, NULL); // Cleanup. Restore the QV's. Delete temporary storage. for (uint32 ii=0; iiget(0); sprintf(name, "%s/reads", _storePath); _readsMMap = new memoryMappedFile (name, memoryMappedFile_readOnly); _reads = (gkRead *)_readsMMap->get(0); sprintf(name, "%s/blobs", _storePath); _blobsMMap = new memoryMappedFile (name, memoryMappedFile_readOnly); _blobs = (void *)_blobsMMap->get(0); } // // MODIFY, NO APPEND (also for building a partitioned store) // else if ((mode == gkStore_modify) && (partID == UINT32_MAX)) { //fprintf(stderr, "gkStore()-- opening '%s' for read-write access.\n", _storePath); if (AS_UTL_fileExists(_storePath, true, false) == false) { fprintf(stderr, "gkStore()-- failed to open '%s' for read-write access: store doesn't exist.\n", _storePath); exit(1); } sprintf(name, "%s/libraries", _storePath); _librariesMMap = new memoryMappedFile (name, memoryMappedFile_readWrite); _libraries = (gkLibrary *)_librariesMMap->get(0); sprintf(name, "%s/reads", _storePath); _readsMMap = new memoryMappedFile (name, memoryMappedFile_readWrite); _reads = (gkRead *)_readsMMap->get(0); sprintf(name, "%s/blobs", _storePath); _blobsMMap = new memoryMappedFile (name, memoryMappedFile_readWrite); _blobs = (void *)_blobsMMap->get(0); } // // MODIFY, APPEND, open mmap'd files, but copy them entirely to local memory // else if ((mode == gkStore_extend) && (partID == UINT32_MAX)) { //fprintf(stderr, "gkStore()-- opening '%s' for read-write and append access.\n", _storePath); if (AS_UTL_fileExists(_storePath, true, true) == false) AS_UTL_mkdir(_storePath); _librariesAlloc = MAX(64, 2 * _info.numLibraries); _libraries = new gkLibrary [_librariesAlloc]; sprintf(name, "%s/libraries", _storePath); if (AS_UTL_fileExists(name, false, false) == true) { _librariesMMap = new memoryMappedFile (name, memoryMappedFile_readOnly); memcpy(_libraries, _librariesMMap->get(0), sizeof(gkLibrary) * (_info.numLibraries + 1)); delete _librariesMMap; _librariesMMap = NULL;; } _readsAlloc = MAX(128, 2 * _info.numReads); _reads = new gkRead [_readsAlloc]; sprintf(name, "%s/reads", _storePath); if (AS_UTL_fileExists(name, false, false) == true) { _readsMMap = new memoryMappedFile (name, memoryMappedFile_readOnly); memcpy(_reads, _readsMMap->get(0), sizeof(gkRead) * (_info.numReads + 1)); delete _readsMMap; _readsMMap = NULL; } sprintf(name, "%s/blobs", _storePath); _blobsMMap = NULL; _blobs = NULL; errno = 0; _blobsFile = fopen(name, "a+"); if (errno) fprintf(stderr, "gkStore()-- Failed to open blobs file '%s' for appending: %s\n", name, strerror(errno)), exit(1); } // // PARTITIONED, no modifications, no appends // // BIG QUESTION: do we want to partition the read metadata too, or is it small enough // to load in every job? For now, we load all the metadata. else if ((mode == gkStore_readOnly) && (partID != UINT32_MAX)) { //fprintf(stderr, "gkStore()-- opening '%s' partition '%u' for read-only access.\n", _storePath, partID); // For partitioned reads, we need to have a uint32 map of readID to partitionReadID so we can // lookup the metadata in the partitoned _reads data. This is 4 bytes per read, compared to 24 // bytes for the full meta data. Assuming 100x of 3kb read coverage on human, that's 100 // million reads, so 0.400 GB vs 2.4 GB. sprintf(name, "%s/partitions/map", _storePath); errno = 0; FILE *F = fopen(name, "r"); if (errno) fprintf(stderr, "gkStore::gkStore()-- failed to open '%s' for reading: %s\n", name, strerror(errno)), exit(1); AS_UTL_safeRead(F, &_numberOfPartitions, "gkStore::_numberOfPartitions", sizeof(uint32), 1); _partitionID = partID; _readsPerPartition = new uint32 [_numberOfPartitions + 1]; // No zeroth element in any of these _readIDtoPartitionID = new uint32 [gkStore_getNumReads() + 1]; _readIDtoPartitionIdx = new uint32 [gkStore_getNumReads() + 1]; AS_UTL_safeRead(F, _readsPerPartition, "gkStore::_readsPerPartition", sizeof(uint32), _numberOfPartitions + 1); AS_UTL_safeRead(F, _readIDtoPartitionID, "gkStore::_readIDtoPartitionID", sizeof(uint32), gkStore_getNumReads() + 1); AS_UTL_safeRead(F, _readIDtoPartitionIdx, "gkStore::_readIDtoPartitionIdx", sizeof(uint32), gkStore_getNumReads() + 1); fclose(F); sprintf(name, "%s/libraries", _storePath); _librariesMMap = new memoryMappedFile (name, memoryMappedFile_readOnly); _libraries = (gkLibrary *)_librariesMMap->get(0); //fprintf(stderr, " -- openend '%s' at "F_X64"\n", name, _libraries); sprintf(name, "%s/partitions/reads.%04"F_U32P"", _storePath, partID); _readsMMap = new memoryMappedFile (name, memoryMappedFile_readOnly); _reads = (gkRead *)_readsMMap->get(0); //fprintf(stderr, " -- openend '%s' at "F_X64"\n", name, _reads); sprintf(name, "%s/partitions/blobs.%04"F_U32P"", _storePath, partID); _blobsMMap = new memoryMappedFile (name, memoryMappedFile_readOnly); _blobs = (void *)_blobsMMap->get(0); //fprintf(stderr, " -- openend '%s' at "F_X64"\n", name, _blobs); } // Info only, no access to reads or libraries. else if (mode == gkStore_infoOnly) { //fprintf(stderr, "gkStore()-- opening '%s' for info-only access.\n", _storePath); } else { fprintf(stderr, "gkStore::gkStore()-- invalid mode '%s' with partition ID %u.\n", toString(mode), partID); assert(0); } } gkStore::~gkStore() { char N[FILENAME_MAX]; FILE *F; // Should check that inf on disk is the same as inf in memory, and update if needed. bool needsInfoUpdate = false; // Write N+1 because we write, but don't count, the [0] element. if (_librariesMMap) { delete _librariesMMap; } else if (_libraries) { sprintf(N, "%s/libraries", gkStore_path()); errno = 0; F = fopen(N, "w"); if (errno) fprintf(stderr, "gkStore::~gkStore()-- failed to open '%s' for writing: %s\n", N, strerror(errno)), exit(1); AS_UTL_safeWrite(F, _libraries, "libraries", sizeof(gkLibrary), gkStore_getNumLibraries() + 1); fclose(F); delete [] _libraries; needsInfoUpdate = true; } if (_readsMMap) { delete _readsMMap; } else if (_reads) { sprintf(N, "%s/reads", gkStore_path()); errno = 0; F = fopen(N, "w"); if (errno) fprintf(stderr, "gkStore::~gkStore()-- failed to open '%s' for writing: %s\n", N, strerror(errno)), exit(1); AS_UTL_safeWrite(F, _reads, "reads", sizeof(gkRead), gkStore_getNumReads() + 1); fclose(F); delete [] _reads; needsInfoUpdate = true; } if (needsInfoUpdate) { sprintf(N, "%s/info", gkStore_path()); errno = 0; F = fopen(N, "w"); if (errno) fprintf(stderr, "gkStore::~gkStore()-- failed to open '%s' for writing: %s\n", N, strerror(errno)), exit(1); AS_UTL_safeWrite(F, &_info, "info", sizeof(gkStoreInfo), 1); fclose(F); sprintf(N, "%s/info.txt", gkStore_path()); errno = 0; F = fopen(N, "w"); if (errno) fprintf(stderr, "gkStore::~gkStore()-- failed to open '%s' for writing: %s\n", N, strerror(errno)), exit(1); _info.writeInfoAsText(F); fclose(F); } if (_blobsMMap) delete _blobsMMap; if (_blobsFile) fclose(_blobsFile); delete [] _readIDtoPartitionIdx; delete [] _readIDtoPartitionID; delete [] _readsPerPartition; }; gkLibrary * gkStore::gkStore_addEmptyLibrary(char const *name) { assert(_librariesMMap == NULL); assert(_info.numLibraries <= _librariesAlloc); // Library[0] doesn't exist, see comments in addEmptyRead below. _info.numLibraries++; if (_info.numLibraries == _librariesAlloc) increaseArray(_libraries, _info.numLibraries, _librariesAlloc, 128); // Bullet proof the library name - so we can make files with this prefix. char libname[LIBRARY_NAME_SIZE]; uint32 libnamepos = 0; bool modified = false; bool truncated = false; for (char const *orig=name; *orig; orig++) { if (*orig == '/') { libname[libnamepos++] = '_'; libname[libnamepos++] = '-'; libname[libnamepos++] = '_'; modified = true; } else if (isspace(*orig) == 0) { libname[libnamepos++] = *orig; } else { libname[libnamepos++] = '_'; modified = true; } if (libnamepos >= LIBRARY_NAME_SIZE) { libname[LIBRARY_NAME_SIZE-1] = 0; truncated = true; break; } } libname[libnamepos] = 0; #if 0 if (modified || truncated) fprintf(stderr, "gkStore_addEmptyLibrary()-- added library '%s' (original name '%s')\n", libname, name); else fprintf(stderr, "gkStore_addEmptyLibrary()-- added library '%s'\n", libname); #endif _libraries[_info.numLibraries] = gkLibrary(); strncpy(_libraries[_info.numLibraries]._libraryName, libname, LIBRARY_NAME_SIZE-1); _libraries[_info.numLibraries]._libraryID = _info.numLibraries; return(_libraries + _info.numLibraries); } gkRead * gkStore::gkStore_addEmptyRead(gkLibrary *lib) { assert(_readsMMap == NULL); assert(_info.numReads <= _readsAlloc); assert(_mode != gkStore_readOnly); // We reserve the zeroth read for "null". This is easy to accomplish // here, just pre-increment the number of reads. However, we need to be sure // to iterate up to and including _info.numReads. _info.numReads++; if (_info.numReads == _readsAlloc) increaseArray(_reads, _info.numReads, _readsAlloc, _info.numReads/2); _reads[_info.numReads] = gkRead(); _reads[_info.numReads]._readID = _info.numReads; _reads[_info.numReads]._libraryID = lib->gkLibrary_libraryID(); //fprintf(stderr, "ADD READ %u = %u alloc = %u\n", _info.numReads, _reads[_info.numReads]._readID, _readsAlloc); return(_reads + _info.numReads); } void gkRead::gkRead_copyDataToPartition(void *blobs, FILE **partfiles, uint64 *partfileslen, uint32 partID) { // Stash away the location of the partitioned data assert(partfileslen[partID] == AS_UTL_ftell(partfiles[partID])); // Figure out where the blob actually is, and make sure that it really is a blob uint8 *blob = (uint8 *)blobs + _mPtr; uint32 blobLen = 8 + *((uint32 *)blob + 1); assert(blob[0] == 'B'); assert(blob[1] == 'L'); assert(blob[2] == 'O'); assert(blob[3] == 'B'); // Write the blob to the partition, update the length of the partition AS_UTL_safeWrite(partfiles[partID], blob, "gkRead::gkRead_copyDataToPartition::blob", sizeof(char), blobLen); // Update the read to the new location of the blob in the partitioned data. _mPtr = partfileslen[partID]; _pID = partID; // And finalize by remembering the length. partfileslen[partID] += blobLen; assert(partfileslen[partID] == AS_UTL_ftell(partfiles[partID])); } void gkStore::gkStore_buildPartitions(uint32 *partitionMap) { char name[FILENAME_MAX]; // Store cannot be partitioned already, and it must be readOnly (for safety) as we don't need to // be changing any of the normal store data. assert(_numberOfPartitions == 0); assert(_mode == gkStore_readOnly); // Figure out what the last partition is uint32 maxPartition = 0; uint32 unPartitioned = 0; assert(partitionMap[0] == UINT32_MAX); for (uint32 fi=1; fi<=gkStore_getNumReads(); fi++) { if (partitionMap[fi] == UINT32_MAX) unPartitioned++; else if (maxPartition < partitionMap[fi]) maxPartition = partitionMap[fi]; } fprintf(stderr, "Found "F_U32" unpartitioned reads and maximum partition of "F_U32"\n", unPartitioned, maxPartition); // Create the partitions by opening N copies of the data stores, // and writing data to each. FILE **blobfiles = new FILE * [maxPartition + 1]; uint64 *blobfileslen = new uint64 [maxPartition + 1]; // Offset, in bytes, into the blobs file FILE **readfiles = new FILE * [maxPartition + 1]; uint32 *readfileslen = new uint32 [maxPartition + 1]; // aka _readsPerPartition uint32 *readIDmap = new uint32 [gkStore_getNumReads() + 1]; // aka _readIDtoPartitionIdx // Be nice and put all the partitions in a subdirectory. sprintf(name,"%s/partitions", _storePath); if (AS_UTL_fileExists(name, true, true) == false) AS_UTL_mkdir(name); // Open all the output files -- fail early if we can't open that many files. blobfiles[0] = NULL; blobfileslen[0] = UINT64_MAX; readfiles[0] = NULL; readfileslen[0] = UINT32_MAX; for (uint32 i=1; i<=maxPartition; i++) { sprintf(name,"%s/partitions/blobs.%04d", _storePath, i); errno = 0; blobfiles[i] = fopen(name, "w"); blobfileslen[i] = 0; if (errno) fprintf(stderr, "gkStore::gkStore_buildPartitions()-- ERROR: failed to open partition %u file '%s' for write: %s\n", i, name, strerror(errno)), exit(1); sprintf(name,"%s/partitions/reads.%04d", _storePath, i); errno = 0; readfiles[i] = fopen(name, "w"); readfileslen[i] = 0; if (errno) fprintf(stderr, "gkStore::gkStore_buildPartitions()-- ERROR: failed to open partition %u file '%s' for write: %s\n", i, name, strerror(errno)), exit(1); } // Open the output partition map file -- we might as well fail early if we can't make it also. sprintf(name,"%s/partitions/map", _storePath); errno = 0; FILE *rIDmF = fopen(name, "w"); if (errno) fprintf(stderr, "gkStore::gkStore_buildPartitions()-- ERROR: failed to open partition map file '%s': %s\n", name, strerror(errno)), exit(1); // Copy the blob from the master file to the partitioned file, update pointers. readIDmap[0] = UINT32_MAX; // There isn't a zeroth read, make it bogus. for (uint32 fi=1; fi<=gkStore_getNumReads(); fi++) { uint32 pi = partitionMap[fi]; assert(pi != 0); // No zeroth partition, right? if (pi == UINT32_MAX) // Deleted reads are not assigned a partition; skip them continue; // Make a copy of the read, then modify it for the partition, then write it to the partition. // Without the copy, we'd need to update the master record too. gkRead partRead = _reads[fi]; //*gkStore_getRead(fi); partRead.gkRead_copyDataToPartition(_blobs, blobfiles, blobfileslen, pi); #if 1 fprintf(stderr, "read "F_U32"="F_U32" len "F_U32" -- blob master "F_U64" -- to part "F_U32" new read id "F_U32" blob "F_U64"/"F_U64" -- at readIdx "F_U32"\n", fi, _reads[fi].gkRead_readID(), _reads[fi].gkRead_sequenceLength(), _reads[fi]._mPtr, pi, partRead.gkRead_readID(), partRead._pID, partRead._mPtr, readfileslen[pi]); #endif AS_UTL_safeWrite(readfiles[pi], &partRead, "gkStore::gkStore_buildPartitions::read", sizeof(gkRead), 1); readIDmap[fi] = readfileslen[pi]++; } // There isn't a zeroth read. AS_UTL_safeWrite(rIDmF, &maxPartition, "gkStore::gkStore_buildPartitions::maxPartition", sizeof(uint32), 1); AS_UTL_safeWrite(rIDmF, readfileslen, "gkStore::gkStore_buildPartitions::readfileslen", sizeof(uint32), maxPartition + 1); AS_UTL_safeWrite(rIDmF, partitionMap, "gkStore::gkStore_buildPartitions::partitionMap", sizeof(uint32), gkStore_getNumReads() + 1); AS_UTL_safeWrite(rIDmF, readIDmap, "gkStore::gkStore_buildPartitions::readIDmap", sizeof(uint32), gkStore_getNumReads() + 1); // cleanup -- close all the files, delete storage fclose(rIDmF); for (uint32 i=1; i<=maxPartition; i++) { fprintf(stderr, "partition "F_U32" has "F_U32" reads\n", i, readfileslen[i]); errno = 0; fclose(blobfiles[i]); fclose(readfiles[i]); if (errno) fprintf(stderr, " warning: %s\n", strerror(errno)); } delete [] readIDmap; delete [] readfileslen; delete [] readfiles; delete [] blobfileslen; delete [] blobfiles; } void gkStore::gkStore_delete(void) { char path[FILENAME_MAX]; delete [] _libraries; delete [] _reads; _libraries = NULL; _reads = NULL; gkStore_deletePartitions(); sprintf(path, "%s/info", gkStore_path()); AS_UTL_unlink(path); sprintf(path, "%s/libraries", gkStore_path()); AS_UTL_unlink(path); sprintf(path, "%s/reads", gkStore_path()); AS_UTL_unlink(path); sprintf(path, "%s/blobs", gkStore_path()); AS_UTL_unlink(path); AS_UTL_unlink(path); } void gkStore::gkStore_deletePartitions(void) { char path[FILENAME_MAX]; sprintf(path, "%s/partitions/map", gkStore_path()); if (AS_UTL_fileExists(path, false, false) == false) return; // How many partitions? FILE *F = fopen(path, "r"); if (errno) fprintf(stderr, "ERROR: failed to open partition meta data '%s': %s\n", path, strerror(errno)), exit(1); AS_UTL_safeRead(F, &_numberOfPartitions, "gkStore_deletePartitions::numberOfPartitions", sizeof(uint32), 1); fclose(F); // Yay! Delete! AS_UTL_unlink(path); for (uint32 ii=0; ii<_numberOfPartitions; ii++) { sprintf(path, "%s/partitions/reads.%04u", gkStore_path(), ii+1); AS_UTL_unlink(path); sprintf(path, "%s/partitions/blobs.%04u", gkStore_path(), ii+1); AS_UTL_unlink(path); } } void gkStoreStats::init(gkStore *UNUSED(gkp)) { #if 0 gkFragment fr; gkStream *fs = new gkStream(gkp, 0, 0, GKFRAGMENT_INF); numActiveFrag = 0; numMatedFrag = 0; readLength = 0; clearLength = 0; lowestID = new uint32 [gkp->gkStore_getNumLibraries() + 1]; highestID = new uint32 [gkp->gkStore_getNumLibraries() + 1]; numActivePerLib = new uint32 [gkp->gkStore_getNumLibraries() + 1]; numMatedPerLib = new uint32 [gkp->gkStore_getNumLibraries() + 1]; readLengthPerLib = new uint64 [gkp->gkStore_getNumLibraries() + 1]; clearLengthPerLib = new uint64 [gkp->gkStore_getNumLibraries() + 1]; for (uint32 i=0; igkStore_getNumLibraries() + 1; i++) { lowestID[i] = 0; highestID[i] = 0; numActivePerLib[i] = 0; numMatedPerLib[i] = 0; readLengthPerLib[i] = 0; clearLengthPerLib[i] = 0; } while (fs->next(&fr)) { uint32 lid = fr.gkFragment_getLibraryID(); uint32 rid = fr.gkFragment_getReadID(); if (lowestID[lid] == 0) { lowestID[lid] = rid; highestID[lid] = rid; } if (highestID[lid] < rid) { highestID[lid] = rid; } numActiveFrag++; numActivePerLib[lid]++; readLength += fr.gkFragment_getSequenceLength(); readLengthPerLib[lid] += fr.gkFragment_getSequenceLength(); clearLength += fr.gkFragment_getClearRegionLength(); clearLengthPerLib[lid] += fr.gkFragment_getClearRegionLength(); } delete fs; #endif } canu-1.3/src/stores/gkStore.H000066400000000000000000000504151272335422300161500ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2014-NOV-26 to 2015-AUG-14 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-09 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef GKSTORE_H #define GKSTORE_H #include "AS_global.H" #include "memoryMappedFile.H" #include using namespace std; // The number of library IIDs we can handle. // #define AS_MAX_LIBRARIES_BITS 6 #define AS_MAX_LIBRARIES (((uint32)1 << AS_MAX_LIBRARIES_BITS) - 1) #define LIBRARY_NAME_SIZE 128 // Maximum length of reads. // // If 16, an overlap is only 20 bytes. (5x 32 bit words) // If 17-21, an overlap is 24 bytes. (3x 64 bit words) // If 22-32, an overlap is 32 bytes. (4x 64 bit words) // // if 26, bogart has issues with storing the error rate // If 28, alignment/alignment-drivers.C won't link // If 29, alignment/alignment-drivers.C won't link // If 30, alignment/alignment-drivers.C won't link // If 31, alignment/alignment-drivers.C won't compile, len+len+2 == 0 // If 32, it won't compile because of shifting (uint32)1 << 32 == 0. // #define AS_MAX_READLEN_BITS 21 #define AS_MAX_READLEN (((uint32)1 << AS_MAX_READLEN_BITS) - 1) // The number of read IDs we can handle. Longer reads implies fewer reads. // readLen 32 + numLibs 6 -> numReads 26 ( 64 million) // readLen 30 + numLibs 6 -> numReads 28 (256 million) // readLen 28 + numLibs 6 -> numReads 30 (1024 million) // readLen 26 + numLibs 6 -> numReads 32 (4096 million) // limited elsewhere! // readLen 24 + numLibs 6 -> numReads 34 (4096 million) // limited elsewhere! // readLen 22 + numLibs 6 -> numReads 36 (4096 million) // limited elsewhere! // readLen 21 + numLibs 6 -> numReads 37 (4096 million) // limited elsewhere! // readLen 20 + numLibs 6 -> numReads 38 (4096 million) // limited elsewhere! // #define AS_MAX_READS_BITS 64 - AS_MAX_READLEN_BITS - AS_MAX_LIBRARIES_BITS #define AS_MAX_READS (((uint64)1 << AS_MAX_READS_BITS) - 1) // Per-library options. // Read type #define GK_READTYPE_GENERIC 0x0000 #define GK_READTYPE_CONTIG 0x0001 #define GK_READTYPE_PACBIO_RAW 0x0002 #define GK_READTYPE_PACBIO_CORRECTED 0x0003 #define GK_READTYPE_NANOPORE_RAW 0x0004 #define GK_READTYPE_NANOPORE_CORRECTED 0x0005 // Correction algorithm #define GK_CORRECTION_NONE 0x0000 #define GK_CORRECTION_CONSENSUS 0x0001 #define GK_CORRECTION_MER 0x0002 // Trimming algorithm #define GK_FINALTRIM_NONE 0x0000 #define GK_FINALTRIM_LARGEST_COVERED 0x0001 // largest region covered by good overlaps #define GK_FINALTRIM_BEST_EDGE 0x0002 // largest region covered by best overlaps (broken) class gkLibrary { public: gkLibrary() { memset(_libraryName, 0, sizeof(char) * LIBRARY_NAME_SIZE); strncpy(_libraryName, "UNDEFINED", LIBRARY_NAME_SIZE-1); gkLibrary_clearFeatures(); }; gkLibrary(char const *name) { memset(_libraryName, 0, sizeof(char) * LIBRARY_NAME_SIZE); strncpy(_libraryName, name, LIBRARY_NAME_SIZE-1); gkLibrary_clearFeatures(); }; ~gkLibrary() { }; void gkLibrary_clearFeatures(void) { // DO NOT change defaults without updating gkLibrary_parsePreset(). _isNonRandom = 0; _readCorrection = GK_CORRECTION_NONE; _readType = GK_READTYPE_GENERIC; _finalTrim = GK_FINALTRIM_LARGEST_COVERED; _removeDuplicateReads = 1; _removeSpurReads = 1; _removeChimericReads = 1; _checkForSubReads = 1; _defaultQV = 20; }; public: char const *gkLibrary_libraryName(void) { return(_libraryName); }; uint32 gkLibrary_libraryID(void) { return(_libraryID); }; uint32 gkLibrary_isNonRandom(void) { return(_isNonRandom); }; uint32 gkLibrary_readType(void) { return(_readType); }; char const *gkLibrary_readTypeString(void); uint32 gkLibrary_readCorrection(void) { return(_readCorrection); }; char const *gkLibrary_readCorrectionString(void); uint32 gkLibrary_finalTrim(void) { return(_finalTrim); }; char const *gkLibrary_finalTrimString(void); uint32 gkLibrary_removeDuplicateReads(void) { return(_removeDuplicateReads); }; uint32 gkLibrary_removeSpurReads(void) { return(_removeSpurReads); }; uint32 gkLibrary_removeChimericReads(void) { return(_removeChimericReads); }; uint32 gkLibrary_checkForSubReads(void) { return(_checkForSubReads); }; uint32 gkLibrary_defaultQV(void) { return(_defaultQV); }; void gkLibrary_setIsNonRandom(bool f) { _isNonRandom = f; }; void gkLibrary_setReadType(char *f); void gkLibrary_setReadCorrection(char *t); void gkLibrary_setFinalTrim(char *t); void gkLibrary_setRemoveDuplicateReads(bool f) { _removeDuplicateReads = f; }; void gkLibrary_setRemoveSpurReads(bool f) { _removeSpurReads = f; }; void gkLibrary_setRemoveChimericReads(bool f) { _removeChimericReads = f; }; void gkLibrary_setCheckForSubReads(bool f) { _checkForSubReads = f; }; void gkLibrary_setDefaultQV(double qv) { _defaultQV = qv; }; void gkLibrary_parsePreset(char *t); private: char _libraryName[LIBRARY_NAME_SIZE]; uint32 _libraryID; // If set, reads are from a non-random library and shouldn't count toward coverage stats. uint32 _isNonRandom; // What generated these reads? uint32 _readType; // Should reads be corrected? How? uint32 _readCorrection; // Should reads be trimmed based on overlaps? How? uint32 _finalTrim; // Should duplicate reads (based on overlaps) be removed? uint32 _removeDuplicateReads; // Should spur reads be cleaned up? How? uint32 _removeSpurReads; // Should chimeric reads be cleaned up? How? uint32 _removeChimericReads; // Should PacBio circular sub-reads be cleaned up? How? uint32 _checkForSubReads; // For reads with no QVs, use this. uint32 _defaultQV; friend class gkStore; }; class gkRead; class gkReadData { public: gkReadData() { _read = NULL; _seq = NULL; _qlt = NULL; _seqAlloc = 0; _blobLen = 0; _blobMax = 0; _blob = NULL; _pacbio._deletionQV = NULL; _pacbio._deletionTag = NULL; _pacbio._insertionQV = NULL; _pacbio._substitutionQV = NULL; _pacbio._mergeQV = NULL; _nanopore._someQV = NULL; }; ~gkReadData() { delete [] _seq; delete [] _qlt; delete [] _blob; delete [] _pacbio._deletionQV; delete [] _pacbio._deletionTag; delete [] _pacbio._insertionQV; delete [] _pacbio._substitutionQV; delete [] _pacbio._mergeQV; delete [] _nanopore._someQV; }; gkRead *gkReadData_getRead(void) { return(_read); }; bool gkReadData_hasSequence(void) { return(_seq != NULL); }; bool gkReadData_hasQualities(void) { return(_qlt != NULL); }; char *gkReadData_getSequence(void) { return(_seq); }; char *gkReadData_getQualities(void) { return(_qlt); }; private: gkRead *_read; // Pointer to the mmap'd read char *_seq; // Everyone has sequence char *_qlt; // and quality uint32 _seqAlloc; uint32 _blobLen; uint32 _blobMax; uint8 *_blob; // And maybe even an encoded blob of data from the store. // Used by the store for adding a read. void gkReadData_encodeBlobChunk(char const *tag, uint32 len, void *dat); // Some reads have more data than others. This used to be a union, but that // only complicated things with no clear beneift. Yay, we saved a few bytes on // a structure that we have only a few copies in core at a time. struct pacbio { char *_deletionQV; char *_deletionTag; char *_insertionQV; char *_substitutionQV; char *_mergeQV; } _pacbio; struct nanopore { char *_someQV; } _nanopore; friend class gkRead; friend class gkStore; }; class gkRead { public: gkRead() { _readID = 0; _libraryID = 0; _seqLen = 0; _mPtr = 0; _pID = 0; }; ~gkRead() { }; uint32 gkRead_readID(void) { return((uint32)_readID); }; uint32 gkRead_libraryID(void) { return((uint32)_libraryID); }; uint32 gkRead_sequenceLength(void) { return((uint32)_seqLen); }; uint64 gkRead_mPtr(void) { return(_mPtr); }; // For debugging, in gatekeeperDumpMetatData uint64 gkRead_pID(void) { return(_pID); }; public: bool gkRead_loadData(gkReadData *readData, void *blob); private: uint32 gkRead_encode2bit(uint8 *&chunk, char *seq, uint32 seqLen); uint32 gkRead_encode3bit(uint8 *&chunk, char *seq, uint32 seqLen); uint32 gkRead_encode4bit(uint8 *&chunk, char *qlt, uint32 seqLen); uint32 gkRead_encode5bit(uint8 *&chunk, char *qlt, uint32 seqLen); bool gkRead_decode2bit(uint8 *chunk, uint32 chunkLen, char *seq, uint32 seqLen); bool gkRead_decode3bit(uint8 *chunk, uint32 chunkLen, char *seq, uint32 seqLen); bool gkRead_decode4bit(uint8 *chunk, uint32 chunkLen, char *qlt, uint32 seqLen); bool gkRead_decode5bit(uint8 *chunk, uint32 chunkLen, char *qlt, uint32 seqLen); public: // These are called by gatekeeperCreate. gkReadData *gkRead_encodeSeqQlt(char *H, char *S, char *Q, uint32 qv); //gkReadData *gkRead_encodePacBio(char *H, char *S, char *Q); // Not implemented. //gkReadData *gkRead_encodeMinION(char *H, char *S, char *Q); private: char *gkRead_encodeSequence(char *sequence, char *encoded); char *gkRead_decodeSequence(char *encoded, char *sequence); char *gkRead_encodeQuality(char *sequence, char *encoded); char *gkRead_decodeQuality(char *encoded, char *sequence); private: // Used by the store to copy data to a partition void gkRead_copyDataToPartition(void *blobs, FILE **partfiles, uint64 *partfileslen, uint32 partID); private: uint64 _readID : AS_MAX_READS_BITS; uint64 _libraryID : AS_MAX_LIBRARIES_BITS; uint64 _seqLen : AS_MAX_READLEN_BITS; uint64 _mPtr : 48; // Pointer to blob of data in blob file, 0..256 TB uint64 _pID : 16; // Partition file id, 0...65536 friend class gkStore; }; // gkStoreInfo is saved on disk. // gkStore is the in memory structure used to access the data. // class gkStoreInfo { public: gkStoreInfo() { gkMagic = 0x504b473a756e6163llu; // canu:GKP gkVersion = 0x0000000000000001llu; gkLibrarySize = sizeof(gkLibrary); gkReadSize = sizeof(gkRead); gkMaxLibrariesBits = AS_MAX_LIBRARIES_BITS; gkLibraryNameSize = LIBRARY_NAME_SIZE; gkMaxReadBits = AS_MAX_READS_BITS; gkMaxReadLenBits = AS_MAX_READLEN_BITS; gkUNUSED = 0; numLibraries = 0; numReads = 0; }; ~gkStoreInfo() { }; void writeInfoAsText(FILE *F) { fprintf(F, "gkMagic = 0x"F_X64"\n", gkMagic); fprintf(F, "gkVersion = 0x"F_X64"\n", gkVersion); fprintf(F, "\n"); fprintf(F, "gkLibrarySize = "F_U32"\n", gkLibrarySize); fprintf(F, "gkReadSize = "F_U32"\n", gkReadSize); fprintf(F, "gkMaxLibrariesBits = "F_U32"\n", gkMaxLibrariesBits); fprintf(F, "gkLibraryNameSize = "F_U32"\n", gkLibraryNameSize); fprintf(F, "gkMaxReadBits = "F_U32"\n", gkMaxReadBits); fprintf(F, "gkMaxReadLenBits = "F_U32"\n", gkMaxReadLenBits); fprintf(F, "\n"); fprintf(F, "numLibraries = "F_U32"\n", numLibraries); fprintf(F, "numReads = "F_U32"\n", numReads); }; private: uint64 gkMagic; uint64 gkVersion; uint32 gkLibrarySize; // Sanity checks that this code can load the data properly. uint32 gkReadSize; uint32 gkMaxLibrariesBits; uint32 gkLibraryNameSize; uint32 gkMaxReadBits; uint32 gkMaxReadLenBits; uint32 gkUNUSED; // Used to hold a blob block size that was never implemented uint32 numLibraries; // Counts of types of things we have loaded uint32 numReads; friend class gkStore; }; // The default behavior is to open the store for read only, and to load // all the metadata into memory. typedef enum { gkStore_readOnly = 0x00, // Open read only gkStore_modify = 0x01, // Open for modification gkStore_extend = 0x02, // Open for modification and appending new reads/libraries gkStore_infoOnly = 0x03 // Open read only, but only load the info on the store; no access to reads or libraries } gkStore_mode; static const char * toString(gkStore_mode m) { switch (m) { case gkStore_readOnly: return("gkStore_readOnly"); break; case gkStore_modify: return("gkStore_modify"); break; case gkStore_extend: return("gkStore_extend"); break; case gkStore_infoOnly: return("gkStore_infoOnly"); break; } return("undefined-mode"); } class gkStore { private: gkStore(char const *path, gkStore_mode mode, uint32 partID); ~gkStore(); public: static gkStore *gkStore_open(char const *path, gkStore_mode mode=gkStore_readOnly, uint32 partID=UINT32_MAX) { #pragma omp critical { // If an instance exists, return it. if (_instance != NULL) { _instanceCount++; //fprintf(stderr, "gkStore_open(%s), %u instances now\n", path, _instanceCount); } // Otherwise, make a new one. else { _instance = new gkStore(path, mode, partID); _instanceCount = 1; //fprintf(stderr, "gkStore_open(%s), first instance, create store\n", path); } } return(_instance); }; void gkStore_close(void) { #pragma omp critical { _instanceCount--; if (_instanceCount == 0) { delete _instance; _instance = NULL; //fprintf(stderr, "gkStore_close(%s), no instances remain, delete store\n", _storeName, _instanceCount); } else { //fprintf(stderr, "gkStore_close(%s), %u instances remain\n", _storeName, _instanceCount); } } }; public: const char *gkStore_path(void) { return(_storePath); }; // Returns the path to the store const char *gkStore_name(void) { return(_storeName); }; // Returns the name, e.g., name.gkpStore void gkStore_buildPartitions(uint32 *partitionMap); void gkStore_delete(void); // Deletes the files in the store. void gkStore_deletePartitions(void); // Deletes the files for a partition. uint32 gkStore_getNumLibraries(void) { return(_info.numLibraries); }; uint32 gkStore_getNumReads(void) { return(_info.numReads); }; gkLibrary *gkStore_getLibrary(uint32 id) { return(&_libraries[id]); }; // Returns a read, using the copy in the partition if the partition exists. gkRead *gkStore_getRead(uint32 id) { if ((_readIDtoPartitionID) && (_readIDtoPartitionID[id] != _partitionID)) { fprintf(stderr, "getRead()-- WARNING: access to read %u in partition %u is slow when partition %u is loaded.\n", id, _readIDtoPartitionID[id], _partitionID); assert(0); } if ((_readIDtoPartitionID) && (_readIDtoPartitionID[id] == _partitionID)) { //fprintf(stderr, "getRead()-- id=%u mapped=%u mappedid=%u\n", // id, _readIDtoPartitionIdx[id], _reads[_readIDtoPartitionIdx[id]].gkRead_readID()); return(_reads + _readIDtoPartitionIdx[id]); } return(_reads + id); }; // Returns a read, but only if it is in the currently loaded partition. gkRead *gkStore_getReadInPartition(uint32 id) { if (_readIDtoPartitionID == NULL) // Not partitioned, return regular read. return(gkStore_getRead(id)); if (_readIDtoPartitionID[id] != _partitionID) // Patitioned, and not in this partition. return(NULL); //fprintf(stderr, "getRead()-- SUCCESS: access to read %u in partition %u is FAST when partition %u is loaded.\n", // id, _readIDtoPartitionID[id], _partitionID); return(_reads + _readIDtoPartitionIdx[id]); } gkLibrary *gkStore_addEmptyLibrary(char const *name); gkRead *gkStore_addEmptyRead(gkLibrary *lib); bool gkStore_loadReadData(gkRead *read, gkReadData *readData) { return(read->gkRead_loadData(readData, _blobs)); }; bool gkStore_loadReadData(uint32 readID, gkReadData *readData) { return(gkStore_getRead(readID)->gkRead_loadData(readData, _blobs)); }; void gkStore_stashReadData(gkRead *read, gkReadData *data); static void gkStore_loadReadFromStream(FILE *S, gkRead *read, gkReadData *readData); void gkStore_saveReadToStream(FILE *S, uint32 id); private: static gkStore *_instance; static uint32 _instanceCount; gkStoreInfo _info; // All the stuff stored on disk. char _storePath[FILENAME_MAX]; // Needed to create files char _storeName[FILENAME_MAX]; // Useful for log files in other programs gkStore_mode _mode; // What mode this store is opened as, sanity checking // If these are memory mapped, then multiple processes on the same host can share the // (read-only) data. memoryMappedFile *_librariesMMap; uint32 _librariesAlloc; // If zero, the mmap is used. gkLibrary *_libraries; memoryMappedFile *_readsMMap; uint32 _readsAlloc; // If zero, the mmap is used. gkRead *_reads; memoryMappedFile *_blobsMMap; // Either the full blobs, or the partitioned blobs. void *_blobs; FILE *_blobsFile; // If the store is openend partitioned, this data is loaded from disk uint32 _numberOfPartitions; // Total number of partitions that exist uint32 _partitionID; // Which partition this is uint32 *_readsPerPartition; // Number of reads in each partition, mostly sanity checking uint32 *_readIDtoPartitionIdx; // Map from global ID to local partition index uint32 *_readIDtoPartitionID; // Map from global ID to partition ID }; class gkStoreStats { public: gkStoreStats(char const *gkStoreName) { gkStore *gkp = gkStore::gkStore_open(gkStoreName); init(gkp); gkp->gkStore_close(); }; gkStoreStats(gkStore *gkp) { init(gkp); }; ~gkStoreStats() { delete [] lowestID; delete [] highestID; delete [] numActivePerLib; delete [] readLengthPerLib; delete [] clearLengthPerLib; }; void init(gkStore *gkp); // Global stats over the whole store uint32 numActiveFrag; uint64 readLength; uint64 clearLength; // Per library stats uint32 *lowestID; uint32 *highestID; uint32 *numActivePerLib; uint64 *readLengthPerLib; uint64 *clearLengthPerLib; }; #endif canu-1.3/src/stores/gkStoreEncode.C000066400000000000000000000102451272335422300172560ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2015-DEC-03 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "gkStore.H" // Encode seq as 2-bit bases. Doesn't touch qlt. uint32 gkRead::gkRead_encode2bit(uint8 *&chunk, char *seq, uint32 seqLen) { // Scan the read, if there are non-acgt, return length 0; this cannot encode it. for (uint32 ii=0; ii> 6) & 0x03)]; seq[ii++] = acgt[((byte >> 4) & 0x03)]; seq[ii++] = acgt[((byte >> 2) & 0x03)]; seq[ii++] = acgt[((byte >> 0) & 0x03)]; } else { if (ii < seqLen) seq[ii++] = acgt[((byte >> 6) & 0x03)]; // This if is also redundant, and also pretty. if (ii < seqLen) seq[ii++] = acgt[((byte >> 4) & 0x03)]; if (ii < seqLen) seq[ii++] = acgt[((byte >> 2) & 0x03)]; if (ii < seqLen) seq[ii++] = acgt[((byte >> 0) & 0x03)]; } } seq[seqLen] = 0; return(true); } // Encode seq as 3-bases-in-7-bits. Doesn't touch qlt. uint32 gkRead::gkRead_encode3bit(uint8 *&UNUSED(chunk), char *UNUSED(seq), uint32 UNUSED(seqLen)) { return(0); } bool gkRead::gkRead_decode3bit(uint8 *UNUSED(chunk), uint32 UNUSED(chunkLen), char *UNUSED(seq), uint32 UNUSED(seqLen)) { return(false); } // Encode qualities as 4 bit integers. Doesn't touch seq. uint32 gkRead::gkRead_encode4bit(uint8 *&UNUSED(chunk), char *qlt, uint32 UNUSED(seqLen)) { if (qlt[0] == 0) // No QVs in the string. return(0); return(0); } bool gkRead::gkRead_decode4bit(uint8 *UNUSED(chunk), uint32 UNUSED(chunkLen), char *UNUSED(qlt), uint32 UNUSED(seqLen)) { return(false); } // Encode qualities as 5 bit integers. Doesn't touch seq. uint32 gkRead::gkRead_encode5bit(uint8 *&UNUSED(chunk), char *qlt, uint32 UNUSED(seqLen)) { if (qlt[0] == 0) // No QVs in the string. return(0); return(0); } bool gkRead::gkRead_decode5bit(uint8 *UNUSED(chunk), uint32 UNUSED(chunkLen), char *UNUSED(qlt), uint32 UNUSED(seqLen)) { return(false); } canu-1.3/src/stores/ovOverlap.C000066400000000000000000000116041272335422300164770ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2014-DEC-15 to 2015-JUL-01 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Sergey Koren beginning on 2016-MAR-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "ovStore.H" #include "gkStore.H" // Even though the b_end_hi | b_end_lo is uint64 in the struct, the result // of combining them doesn't appear to be 64-bit. The cast is necessary. char * ovOverlap::toString(char *str, ovOverlapDisplayType type, bool newLine) { switch (type) { case ovOverlapAsHangs: sprintf(str, "%10"F_U32P" %10"F_U32P" %c %6"F_S32P" %6"F_U32P" %6"F_S32P" %7.6f%s%s", a_iid, b_iid, flipped() ? 'I' : 'N', a_hang(), span(), b_hang(), erate(), (overlapIsDovetail()) ? "" : " PARTIAL", (newLine) ? "\n" : ""); break; case ovOverlapAsCoords: sprintf(str, "%10"F_U32P" %10"F_U32P" %c %6"F_U32P" %6"F_U32P" %6"F_U32P" %6"F_U32P" %6"F_U32P" %7.6f%s", a_iid, b_iid, flipped() ? 'I' : 'N', span(), a_bgn(), a_end(), b_bgn(), b_end(), erate(), (newLine) ? "\n" : ""); break; case ovOverlapAsRaw: sprintf(str, "%10"F_U32P" %10"F_U32P" %c %6"F_U32P" %6"F_U64P" %6"F_U64P" %6"F_U64P" %6"F_U64P" %7.6f %s %s %s%s", a_iid, b_iid, flipped() ? 'I' : 'N', span(), dat.ovl.ahg5, dat.ovl.ahg3, dat.ovl.bhg5, dat.ovl.bhg3, erate(), dat.ovl.forOBT ? "OBT" : " ", dat.ovl.forDUP ? "DUP" : " ", dat.ovl.forUTG ? "UTG" : " ", (newLine) ? "\n" : ""); break; case ovOverlapAsCompat: sprintf(str, "%8"F_U32P" %8"F_U32P" %c %6d %6d %5.2f %5.2f%s", a_iid, b_iid, dat.ovl.flipped ? 'I' : 'N', a_hang(), b_hang(), erate() * 100.0, erate() * 100.0, (newLine) ? "\n" : ""); break; case ovOverlapAsPaf: // miniasm/map expects entries to be separated by tabs // no padding spaces on names we don't confuse read identifiers sprintf(str, "%"F_U32P"\t%6"F_U32P"\t%6"F_U32P"\t%6"F_U32P"\t%c\t%"F_U32P"\t%6"F_U32P"\t%6"F_U32P"\t%6"F_U32P"\t%6"F_U32P"\t%6"F_U32P"\t%6"F_U32P" %s", a_iid, (g->gkStore_getRead(a_iid)->gkRead_sequenceLength()), a_bgn(), a_end(), flipped() ? '-' : '+', b_iid, (g->gkStore_getRead(b_iid)->gkRead_sequenceLength()), flipped() ? b_end() : b_bgn(), flipped() ? b_bgn() : b_end(), (uint32)floor(span() == 0 ? (1-erate() * (a_end()-a_bgn())) : (1-erate()) * span()), span() == 0 ? a_end() - a_bgn() : span(), 255, (newLine) ? "\n" : ""); break; } return(str); } void ovOverlap::swapIDs(ovOverlap const &orig) { a_iid = orig.b_iid; b_iid = orig.a_iid; // Copy the overlap as is, then fix it for the ID swap. #if (ovOverlapNWORDS == 5) dat.dat[0] = orig.dat.dat[0]; dat.dat[1] = orig.dat.dat[1]; dat.dat[2] = orig.dat.dat[2]; dat.dat[3] = orig.dat.dat[3]; dat.dat[4] = orig.dat.dat[4]; #else dat.dat[0] = orig.dat.dat[0]; dat.dat[1] = orig.dat.dat[1]; dat.dat[2] = orig.dat.dat[2]; #endif // Swap the A and B hangs. If the overlap is flipped, we also need to reverse 5' and 3' hangs to // make the now-A read forward oriented. if (orig.dat.ovl.flipped == false) { dat.ovl.ahg5 = orig.dat.ovl.bhg5; dat.ovl.ahg3 = orig.dat.ovl.bhg3; dat.ovl.bhg5 = orig.dat.ovl.ahg5; dat.ovl.bhg3 = orig.dat.ovl.ahg3; } else { dat.ovl.ahg5 = orig.dat.ovl.bhg3; dat.ovl.ahg3 = orig.dat.ovl.bhg5; dat.ovl.bhg5 = orig.dat.ovl.ahg3; dat.ovl.bhg3 = orig.dat.ovl.ahg5; } // Whatever alignment orientation was in the original, it is opposite now. dat.ovl.alignSwapped = ! orig.dat.ovl.alignSwapped; } canu-1.3/src/stores/ovStore.C000066400000000000000000001272631272335422300161740ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVS/AS_OVS_overlapStore.C * src/AS_OVS/AS_OVS_overlapStore.c * * Modifications by: * * Brian P. Walenz from 2007-MAR-08 to 2013-AUG-01 * are Copyright 2007-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2007-MAY-08 * are Copyright 2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2011-JUN-02 to 2011-JUN-03 * are Copyright 2011 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Gregory Sims from 2012-FEB-01 to 2012-FEB-14 * are Copyright 2012 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-09 to 2015-AUG-14 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-12 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2015-DEC-15 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "ovStore.H" const uint64 ovStoreVersion = 2; const uint64 ovStoreMagic = 0x53564f3a756e6163; // == "canu:OVS - store complete const uint64 ovStoreMagicIncomplete = 0x50564f3a756e6163; // == "canu:OVP - store under construction void ovStore::ovStore_write(void) { AS_UTL_mkdir(_storePath); char name[FILENAME_MAX]; sprintf(name, "%s/info", _storePath); // If the ovs file exists, AND has a valid magic number, then the store is complete and we should // abort before the valid store is destroyed. if (AS_UTL_fileExists(name, false, false)) { errno = 0; FILE *ovsinfo = fopen(name, "r"); if (errno) { fprintf(stderr, "ERROR: failed to read store metadata from '%s': %s\n", name, strerror(errno)); exit(1); } AS_UTL_safeRead(ovsinfo, &_info, "ovStore::ovStore::testinfo", sizeof(ovStoreInfo), 1); fclose(ovsinfo); if (_info._ovsMagic == ovStoreMagic) fprintf(stderr, "ERROR: overlapStore '%s' is a valid overlap store, will not overwrite.\n", _storePath), exit(1); } // Create a new incomplete info file. errno = 0; FILE *ovsinfo = fopen(name, "w"); if (errno) fprintf(stderr, "failed to create overlap store '%s': %s\n", _storePath, strerror(errno)), exit(1); AS_UTL_safeWrite(ovsinfo, &_info, "ovStore::ovStore::saveinfo", sizeof(ovStoreInfo), 1); fclose(ovsinfo); sprintf(name, "%s/index", _storePath); errno = 0; _offtFile = fopen(name, "w"); if (errno) fprintf(stderr, "AS_OVS_createOverlapStore()-- failed to open offset file '%s': %s\n", name, strerror(errno)), exit(1); _overlapsThisFile = 0; _currentFileIndex = 0; _bof = NULL; } void ovStore::ovStore_read(void) { char name[FILENAME_MAX]; sprintf(name, "%s/info", _storePath); errno = 0; FILE *ovsinfo = fopen(name, "r"); if (errno) fprintf(stderr, "ERROR: directory '%s' is not an ovelrapStore; failed to open info file '%s': %s\n", _storePath, name, strerror(errno)), exit(1); AS_UTL_safeRead(ovsinfo, &_info, "ovStore::ovStore::info", sizeof(ovStoreInfo), 1); fclose(ovsinfo); if ((_info._ovsMagic != ovStoreMagic) && (_info._ovsMagic != ovStoreMagicIncomplete)) fprintf(stderr, "ERROR: directory '%s' is not an overlapStore; magic number 0x%016"F_X64P" incorrect.\n", _storePath, _info._ovsMagic), exit(1); if ((_info._ovsMagic != ovStoreMagic) && (_info._ovsMagic != ovStoreMagicIncomplete)) fprintf(stderr, "ERROR: overlapStore '%s' is incomplate; creation crashed?\n", _storePath), exit(1); if (_info._ovsVersion != ovStoreVersion) fprintf(stderr, "ERROR: overlapStore '%s' is version "F_U64"; this code supports only version "F_U64".\n", _storePath, _info._ovsVersion, ovStoreVersion), exit(1); if (_info._maxReadLenInBits != AS_MAX_READLEN_BITS) fprintf(stderr, "ERROR: overlapStore '%s' is for AS_MAX_READLEN_BITS="F_U64"; this code supports only %d bits.\n", _storePath, _info._maxReadLenInBits, AS_MAX_READLEN_BITS), exit(1); // Load stats #if 0 sprintf(name, "%s/statistics", _storePath); errno = 0; FILE *ost = fopen(name, "r"); if (errno) fprintf(stderr, "failed to open the stats file '%s': %s\n", name, strerror(errno)), exit(1); AS_UTL_safeRead(ost, &_stats, "ovStore::ovStore::stats", sizeof(OverlapStoreStats), 1); fclose(ost); #endif // Open the index sprintf(name, "%s/index", _storePath); errno = 0; _offtFile = fopen(name, "r"); if (errno) fprintf(stderr, "ERROR: failed to open offset file '%s': %s\n", name, strerror(errno)), exit(1); // Open erates sprintf(name, "%s/evalues", _storePath); if (AS_UTL_fileExists(name)) { _evaluesMap = new memoryMappedFile(name, memoryMappedFile_readOnly); _evalues = (uint16 *)_evaluesMap->get(0); } } ovStore::ovStore(const char *path, gkStore *gkp, ovStoreType cType) { if (path == NULL) fprintf(stderr, "ovStore::ovStore()-- ERROR: no name supplied.\n"), exit(1); if ((path[0] == '-') && (path[1] == 0)) fprintf(stderr, "ovStore::ovStore()-- ERROR: name cannot be '-' (stdin).\n"), exit(1); memset(_storePath, 0, FILENAME_MAX); strncpy(_storePath, path, FILENAME_MAX-1); _isOutput = (cType & ovStoreWrite) ? true : false; _info._ovsMagic = ovStoreMagicIncomplete; // Appropriate for a new store. _info._ovsVersion = ovStoreVersion; _info._smallestIID = UINT64_MAX; _info._largestIID = 0; _info._numOverlapsTotal = 0; _info._highestFileIndex = 0; _info._maxReadLenInBits = AS_MAX_READLEN_BITS; _offtFile = NULL; _offt.clear(); _offm.clear(); _evaluesMap = NULL; _evalues = NULL; _overlapsThisFile = 0; _currentFileIndex = 0; _bof = NULL; // Now open an existing store, or a create a new store. if (_isOutput == false) ovStore_read(); else ovStore_write(); // AFTER the info is loaded, set the ranges. _firstIIDrequested = _info._smallestIID; _lastIIDrequested = _info._largestIID; _gkp = gkp; } ovStore::~ovStore() { // If output, write the last index element (don't forget to fill in gaps); // update the info, using the final magic number if (_isOutput) { if (_offt._numOlaps > 0) { for (; _offm._a_iid < _offt._a_iid; _offm._a_iid++) { _offm._fileno = _offt._fileno; _offm._offset = _offt._offset; _offm._numOlaps = 0; AS_UTL_safeWrite(_offtFile, &_offm, "ovStore::~ovStore::offm", sizeof(ovStoreOfft), 1); } AS_UTL_safeWrite(_offtFile, &_offt, "ovStore::~ovStore::offt", sizeof(ovStoreOfft), 1); } _info._ovsMagic = ovStoreMagic; _info._ovsVersion = ovStoreVersion; _info._highestFileIndex = _currentFileIndex; char name[FILENAME_MAX]; sprintf(name, "%s/info", _storePath); errno = 0; FILE *ovsinfo = fopen(name, "w"); if (errno) fprintf(stderr, "failed to create overlap store '%s': %s\n", _storePath, strerror(errno)), exit(1); AS_UTL_safeWrite(ovsinfo, &_info, "ovStore::~ovStore::info", sizeof(ovStoreInfo), 1); fclose(ovsinfo); fprintf(stderr, "Closing the new store:\n"); fprintf(stderr, " info._ovsMagic = 0x%016"F_X64P"\n", _info._ovsMagic); fprintf(stderr, " info._ovsVersion = "F_U64"\n", _info._ovsVersion); fprintf(stderr, " info._smallestIID = "F_U64"\n", _info._smallestIID); fprintf(stderr, " info._largestIID = "F_U64"\n", _info._largestIID); fprintf(stderr, " info._numOverlapsTotal = "F_U64"\n", _info._numOverlapsTotal); fprintf(stderr, " info._highestFileIndex = "F_U64"\n", _info._highestFileIndex); fprintf(stderr, " info._maxReadLenInBits = "F_U64"\n", _info._maxReadLenInBits); } if (_evaluesMap) { delete _evaluesMap; _evaluesMap = NULL; _evalues = NULL; } #if 0 if (_statsUpdated) { fprintf(stderr, "Writing new stats.\n"); char name [FILENAME_MAX]; sprintf(name, "%s/ost", _storePath); errno = 0; FILE *ost = fopen(name, "w"); if (errno) fprintf(stderr, "failed to write overlap stats '%s': %s\n", name, strerror(errno)), exit(1); AS_UTL_safeWrite(ost, &_stats, "AS_OVS_closeOverlapStore", sizeof(OverlapStoreStats), 1); fclose(ost); } #endif delete _bof; fclose(_offtFile); } uint32 ovStore::readOverlap(ovOverlap *overlap) { assert(_isOutput == FALSE); // If we've finished reading overlaps for the current a_iid, get // another a_iid. If we hit EOF here, we're all done, no more // overlaps. while (_offt._numOlaps == 0) if (0 == AS_UTL_safeRead(_offtFile, &_offt, "ovStore::readOverlap::offset", sizeof(ovStoreOfft), 1)) return(0); // And if we've exited the range of overlaps requested, return. if (_offt._a_iid > _lastIIDrequested) return(0); while ((_bof == NULL) || (_bof->readOverlap(overlap) == FALSE)) { char name[FILENAME_MAX]; // We read no overlap, open the next file and try again. if (_bof) delete _bof; _currentFileIndex++; sprintf(name, "%s/%04d", _storePath, _currentFileIndex); _bof = new ovFile(name, ovFileNormal); } overlap->a_iid = _offt._a_iid; overlap->g = _gkp; if (_evalues) overlap->evalue(_evalues[_offt._overlapID++]); _offt._numOlaps--; return(1); } uint32 ovStore::numberOfOverlaps(void) { ovOverlap *ovl = NULL; uint32 novl = 0; return(readOverlaps(ovl, novl)); } uint32 ovStore::readOverlaps(ovOverlap *&overlaps, uint32 &maxOverlaps, bool restrictToIID) { int numOvl = 0; assert(_isOutput == FALSE); // If we've finished reading overlaps for the current a_iid, get // another a_iid. If we hit EOF here, we're all done, no more // overlaps. while (_offt._numOlaps == 0) if (0 == AS_UTL_safeRead(_offtFile, &_offt, "ovStore::readOverlaps::offset", sizeof(ovStoreOfft), 1)) return(0); // And if we've exited the range of overlaps requested, return. if (_offt._a_iid > _lastIIDrequested) return(0); // Just a query? Return the number of overlaps we'd want to read if ((overlaps == NULL) || (maxOverlaps == 0)) return(_offt._numOlaps); // Allocate more space, if needed if (maxOverlaps < _offt._numOlaps) { delete [] overlaps; while (maxOverlaps < _offt._numOlaps) maxOverlaps *= 2; overlaps = ovOverlap::allocateOverlaps(_gkp, maxOverlaps); } // Read all the overlaps for this ID. while (((restrictToIID == true) && (_offt._numOlaps > 0)) || ((restrictToIID == false) && (_offt._numOlaps > 0) && (numOvl < maxOverlaps))) { // Read an overlap. If this fails, open the next partition and read from there. while ((_bof == NULL) || (_bof->readOverlap(overlaps + numOvl) == false)) { char name[FILENAME_MAX]; // We read no overlap, open the next file and try again. delete _bof; _bof = NULL; _currentFileIndex++; if (_currentFileIndex > _info._highestFileIndex) // No more files, stop trying to load an overlap. break; sprintf(name, "%s/%04d", _storePath, _currentFileIndex); _bof = new ovFile(name, ovFileNormal); } // If the currentFileIndex is invalid, we ran out of overlaps to load. Don't save that // empty overlap to the list. if (_currentFileIndex <= _info._highestFileIndex) { overlaps[numOvl].a_iid = _offt._a_iid; overlaps[numOvl].g = _gkp; if (_evalues) overlaps[numOvl].evalue(_evalues[_offt._overlapID++]); numOvl++; assert(_offt._numOlaps > 0); _offt._numOlaps--; } // If restrictToIID == false, we're loading all overlaps up to the end of the store, or the // request last IID. If to the end of store, we never read a last 'offset' and so a_iid is // still valid (below lastIIDrequested == infinity) but numOlaps is still zero, and the mail // loop terminates. if (restrictToIID == false) { while (_offt._numOlaps == 0) if (0 == AS_UTL_safeRead(_offtFile, &_offt, "ovStore::readOverlap::offset", sizeof(ovStoreOfft), 1)) break; if (_offt._a_iid > _lastIIDrequested) break; } } // while space for more overlaps, load overlaps assert(numOvl <= maxOverlaps); return(numOvl); } uint32 ovStore::readOverlaps(uint32 iid, ovOverlap *&ovl, uint32 &ovlLen, uint32 &ovlMax) { // Allocate initial space if needed. if (ovl == NULL) { ovlLen = 0; ovlMax = 65 * 1024; ovl = ovOverlap::allocateOverlaps(_gkp, ovlMax); } if (iid < ovl[0].a_iid) // Overlaps loaded are for a future read. return(0); if (iid == ovl[0].a_iid) // Overlaps loaded are for this read, YAY! We assume that ALL overlaps are loaded // for this iid. return(ovlLen); // Until we load the correct overlap, repeat. do { // Count the number of overlaps we would load ovlLen = numberOfOverlaps(); if (ovlLen == 0) // Quit now if there are no overlaps. This simplifies the rest of the loop. return(0); // Allocate space for these overlaps. while (ovlMax < ovlLen) { ovlMax *= 2; delete [] ovl; ovl = ovOverlap::allocateOverlaps(_gkp, ovlMax); } // Load the overlaps ovlLen = readOverlaps(ovl, ovlMax); // If we read overlaps for a fragment after 'iid', we're done. The client will properly save // these overlaps until the iid becomes active. // if (iid < ovl[0].a_iid) return(0); // If we've found the overlaps, we're still done, we also return the number of overlaps. // if (iid == ovl[0].a_iid) return(ovlLen); // On the otherhand, if we read overlaps for a fragment before 'iid', we can either keep // reading until we find the overlaps for this fragment, or jump to the correct spot to read // overlaps. // // The rule is simple. If we're within 50 of the correct IID, keep streaming. Otherwise, make // a jump. setRange() seems to ALWAYS close and open a file, which is somewhat expensive, // especially if the file doesn't actually change. // if (50 < iid - ovl[0].a_iid) setRange(iid, UINT32_MAX); } while (ovl[0].a_iid < iid); // Code can't get here. // If we ran out of overlaps, the first return is used. // If we read past the iid we're looking for, the second return is used. // If we found the overlaps we're looking for, the thrd return is used. // If we didn't find the overlaps, we loop. assert(0); return(0); } void ovStore::setRange(uint32 firstIID, uint32 lastIID) { char name[FILENAME_MAX]; // make the index be one record per read iid, regardless, then we // can quickly grab the correct record, and seek to the start of // those overlaps if (firstIID > _info._largestIID) firstIID = _info._largestIID + 1; if (lastIID >= _info._largestIID) lastIID = _info._largestIID; // If our range is invalid (firstIID > lastIID) we keep going, and // let readOverlap() deal with it. AS_UTL_fseek(_offtFile, (size_t)firstIID * sizeof(ovStoreOfft), SEEK_SET); // Unfortunately, we need to actually read the record to figure out // where to position the overlap stream. If the read fails, we // silently return, letting readOverlap() deal with // the problem. _offt.clear(); // Everything should notice that offsetFile is at EOF and not try // to find overlaps, but, just in case, we set invalid first/last // IIDs. // _firstIIDrequested = firstIID; _lastIIDrequested = lastIID; if (0 == AS_UTL_safeRead(_offtFile, &_offt, "ovStore::setRange::offset", sizeof(ovStoreOfft), 1)) return; _overlapsThisFile = 0; _currentFileIndex = _offt._fileno; delete _bof; sprintf(name, "%s/%04d", _storePath, _currentFileIndex); _bof = new ovFile(name, ovFileNormal); _bof->seekOverlap(_offt._offset); } void ovStore::resetRange(void) { char name[FILENAME_MAX]; rewind(_offtFile); _offt.clear(); _overlapsThisFile = 0; _currentFileIndex = 1; delete _bof; sprintf(name, "%s/%04d", _storePath, _currentFileIndex); _bof = new ovFile(name, ovFileNormal); _firstIIDrequested = _info._smallestIID; _lastIIDrequested = _info._largestIID; } void ovStore::writeOverlap(ovOverlap *overlap) { char name[FILENAME_MAX]; assert(_isOutput == TRUE); if (_offt._a_iid > overlap->a_iid) { // Woah! The last overlap we saw is bigger than the one we have now?! fprintf(stderr, "LAST: a:"F_U32"\n", _offt._a_iid); fprintf(stderr, "THIS: a:"F_U32" b:"F_U32"\n", overlap->a_iid, overlap->b_iid); } assert(_offt._a_iid <= overlap->a_iid); if (_info._smallestIID > overlap->a_iid) _info._smallestIID = overlap->a_iid; if (_info._largestIID < overlap->a_iid) _info._largestIID = overlap->a_iid; // If we don't have an output file yet, or the current file is // too big, open a new file. // if ((_bof) && (_overlapsThisFile >= 1024 * 1024 * 1024 / _bof->recordSize())) { delete _bof; _bof = NULL; _overlapsThisFile = 0; } if (_bof == NULL) { char name[FILENAME_MAX]; _currentFileIndex++; sprintf(name, "%s/%04d", _storePath, _currentFileIndex); _bof = new ovFile(name, ovFileNormalWrite); } // Put the index to disk, filling any gaps // if ((_offt._numOlaps != 0) && (_offt._a_iid != overlap->a_iid)) { while (_offm._a_iid < _offt._a_iid) { _offm._fileno = _offt._fileno; _offm._offset = _offt._offset; _offm._overlapID = _offt._overlapID; // Not needed, but makes life easier AS_UTL_safeWrite(_offtFile, &_offm, "ovStore::writeOverlap::offset", sizeof(ovStoreOfft), 1); _offm._a_iid++; } _offm._a_iid++; // One more, since this iid is not missing -- we write it next! AS_UTL_safeWrite(_offtFile, &_offt, "AS_OVS_writeOverlapToStore offset", sizeof(ovStoreOfft), 1); _offt._numOlaps = 0; // Reset; this new id has no overlaps yet. } // Update the index if this is the first overlap for this a_iid // if (_offt._numOlaps == 0) { _offt._a_iid = overlap->a_iid; _offt._fileno = _currentFileIndex; _offt._offset = _overlapsThisFile; _offt._overlapID = _info._numOverlapsTotal; } //AS_OVS_accumulateStats(ovs, overlap); _bof->writeOverlap(overlap); _offt._numOlaps++; _info._numOverlapsTotal++; _overlapsThisFile++; } void ovStore::writeOverlap(ovOverlap *overlap, uint32 maxOverlapsThisFile) { char name[FILENAME_MAX]; assert(_isOutput == TRUE); _currentFileIndex++; _overlapsThisFile = 0; for (uint64 i=0; i < maxOverlapsThisFile; i++ ) { // All overlaps will be sorted by a_iid if (_offt._a_iid > overlap[i].a_iid) { fprintf(stderr, "LAST: a:"F_U32"\n", _offt._a_iid); fprintf(stderr, "THIS: a:"F_U32" b:"F_U32"\n", overlap[i].a_iid, overlap[i].b_iid); } assert(_offt._a_iid <= overlap[i].a_iid); if (_info._smallestIID > overlap[i].a_iid) _info._smallestIID = overlap[i].a_iid; if (_info._largestIID < overlap[i].a_iid) _info._largestIID = overlap[i].a_iid; // Put the index to disk, filling any gaps if ((_offt._numOlaps != 0) && (_offt._a_iid != overlap[i].a_iid)) { while (_offm._a_iid < _offt._a_iid) { _offm._fileno = _offt._fileno; _offm._offset = _offt._offset; _offm._overlapID = _offt._overlapID; // Not needed, but makes life easier AS_UTL_safeWrite(_offtFile, &_offm, "AS_OVS_writeOverlapToStore offset", sizeof(ovStoreOfft), 1); _offm._a_iid++; } _offm._a_iid++; // One more, since this iid is not missing -- we write it next! AS_UTL_safeWrite(_offtFile, &_offt, "AS_OVS_writeOverlapToStore offset", sizeof(ovStoreOfft), 1); _offt._numOlaps = 0; // Reset; this new id has no overlaps yet. } // Update the index if this is the first overlap for this a_iid if (_offt._numOlaps == 0) { _offt._a_iid = overlap[i].a_iid; _offt._fileno = _currentFileIndex; _offt._offset = _overlapsThisFile; _offt._overlapID = _info._numOverlapsTotal; } _offt._numOlaps++; _info._numOverlapsTotal++; _overlapsThisFile++; } fprintf(stderr,"Done building index for dumpfile %d.\n",_currentFileIndex); } uint64 ovStore::numOverlapsInRange(void) { size_t originalposition = 0; uint64 i = 0; uint64 len = 0; ovStoreOfft *offsets = NULL; uint64 numolap = 0; if (_firstIIDrequested > _lastIIDrequested) return(0); originalposition = AS_UTL_ftell(_offtFile); AS_UTL_fseek(_offtFile, (size_t)_firstIIDrequested * sizeof(ovStoreOfft), SEEK_SET); // Even if we're doing a whole human-size store, this allocation is // (a) temporary and (b) only 512MB. The only current consumer of // this code is FragCorrectOVL.c, which doesn't run on the whole // human, it runs on ~24 pieces, which cuts this down to < 32MB. len = _lastIIDrequested - _firstIIDrequested + 1; offsets = new ovStoreOfft [len]; if (len != AS_UTL_safeRead(_offtFile, offsets, "AS_OVS_numOverlapsInRange", sizeof(ovStoreOfft), len)) { fprintf(stderr, "AS_OVS_numOverlapsInRange()-- short read on offsets!\n"); exit(1); } for (i=0; i _lastIIDrequested) return(NULL); firstFrag = _firstIIDrequested; lastFrag = _lastIIDrequested; size_t originalPosition = AS_UTL_ftell(_offtFile); AS_UTL_fseek(_offtFile, (size_t)_firstIIDrequested * sizeof(ovStoreOfft), SEEK_SET); // Even if we're doing a whole human-size store, this allocation is // (a) temporary and (b) only 512MB. The only current consumer of // this code is FragCorrectOVL.c, which doesn't run on the whole // human, it runs on ~24 pieces, which cuts this down to < 32MB. uint64 len = _lastIIDrequested - _firstIIDrequested + 1; ovStoreOfft *offsets = new ovStoreOfft [len]; uint32 *numolap = new uint32 [len]; uint64 act = AS_UTL_safeRead(_offtFile, offsets, "ovStore::numOverlapsInRange::offsets", sizeof(ovStoreOfft), len); if (len != act) fprintf(stderr, "AS_OVS_numOverlapsPerFrag()-- short read on offsets! Expected len="F_U64" read act="F_U64"\n", len, act), exit(1); for (uint64 i=0; itype() == memoryMappedFile_readOnly)) { fprintf(stderr, "WARNING: closing read-only evalues file.\n"); delete _evaluesMap; _evaluesMap = NULL; _evalues = NULL; } // Remove a bogus evalues file if one exists. if ((AS_UTL_fileExists(name) == true) && (AS_UTL_sizeOfFile(name) != (sizeof(uint16) * _info._numOverlapsTotal))) { fprintf(stderr, "WARNING: existing evalues file is incorrect size: should be "F_U64" bytes, is "F_U64" bytes. Removing.\n", (sizeof(uint16) * _info._numOverlapsTotal), AS_UTL_sizeOfFile(name)); AS_UTL_unlink(name); } // Make a new evalues file if one doesn't exist. if (AS_UTL_fileExists(name) == false) { fprintf(stderr, "Creating evalues file for "F_U64" overlaps.\r", _info._numOverlapsTotal); errno = 0; FILE *F = fopen(name, "w"); if (errno) fprintf(stderr, "Failed to make evalues file '%s': %s\n", name, strerror(errno)), exit(1); uint16 *Z = new uint16 [1048576]; uint64 Zn = 0; memset(Z, 0, sizeof(uint16) * 1048576); while (Zn < _info._numOverlapsTotal) { uint64 S = (Zn + 1048576 < _info._numOverlapsTotal) ? 1048576 : _info._numOverlapsTotal - Zn; AS_UTL_safeWrite(F, Z, "zero evalues", sizeof(uint16), S); Zn += S; fprintf(stderr, "Creating evalues file for "F_U64" overlaps....%07.3f%%\r", _info._numOverlapsTotal, 100.0 * Zn / _info._numOverlapsTotal); } fprintf(stderr, "Creating evalues file for "F_U64" overlaps....%07.3f%%\n", _info._numOverlapsTotal, 100.0 * Zn / _info._numOverlapsTotal); fclose(F); } // Open the evalues file if it isn't already opened if (_evalues == NULL) { _evaluesMap = new memoryMappedFile(name, memoryMappedFile_readWrite); _evalues = (uint16 *)_evaluesMap->get(0); } // Figure out the overlap ID for the first overlap associated with bgnID setRange(bgnID, endID); // Load the evalues from 'evalues' for (uint64 ii=0; iiwriteOverlap(ovls + i); if (offt._a_iid > ovls[i].a_iid) { fprintf(stderr, "LAST: a:"F_U32"\n", offt._a_iid); fprintf(stderr, "THIS: a:"F_U32" b:"F_U32"\n", ovls[i].a_iid, ovls[i].b_iid); } assert(offt._a_iid <= ovls[i].a_iid); info._smallestIID = MIN(info._smallestIID, ovls[i].a_iid); info._largestIID = MAX(info._largestIID, ovls[i].a_iid); // Put the index to disk, filling any gaps if ((offt._numOlaps != 0) && (offt._a_iid != ovls[i].a_iid)) { while (offm._a_iid < offt._a_iid) { offm._fileno = offt._fileno; offm._offset = offt._offset; offm._overlapID = offt._overlapID; // Not needed, but makes life easier offm._numOlaps = 0; AS_UTL_safeWrite(offtFile, &offm, "AS_OVS_writeOverlapToStore offt", sizeof(ovStoreOfft), 1); offm._a_iid++; } // One more, since this iid is not offm -- we write it next! offm._a_iid++; AS_UTL_safeWrite(offtFile, &offt, "AS_OVS_writeOverlapToStore offt", sizeof(ovStoreOfft), 1); offt._overlapID += offt._numOlaps; // The next block of overlaps starts with this ID offt._numOlaps = 0; // The next block has no overlaps yet. } // Update the index if this is the first overlap for this a_iid if (offt._numOlaps == 0) { offt._a_iid = ovls[i].a_iid; offt._fileno = currentFileIndex; offt._offset = overlapsThisFile; } offt._numOlaps++; info._numOverlapsTotal++; overlapsThisFile++; } // Close the output file. delete bof; // Write the final (empty) index entries. while (offm._a_iid < offt._a_iid) { offm._fileno = offt._fileno; offm._offset = offt._offset; offm._overlapID = offt._overlapID; // Not needed, but makes life easier offm._numOlaps = 0; AS_UTL_safeWrite(offtFile, &offm, "AS_OVS_writeOverlapToStore offt", sizeof(ovStoreOfft), 1); offm._a_iid++; } // And the final (real) index entry. We could, but don't need to, update overlapID with the // number of overlaps in this block. AS_UTL_safeWrite(offtFile, &offt, "AS_OVS_writeOverlapToStore offt", sizeof(ovStoreOfft), 1); fclose(offtFile); // In the nasty case that there were no overlaps in this slice, set meaningful smallest and // largest. Well, at least, set non-nonsense smallest and largest. if (overlapsThisFile == 0) { info._smallestIID = 0; info._largestIID = 0; } // Write the info, and some stats for the user. sprintf(name,"%s/%04d.info", storePath, fileID); errno = 0; FILE *F = fopen(name, "w"); if (errno) fprintf(stderr, "ERROR: Failed to open '%s' for writing: %s\n", name, strerror(errno)), exit(1); AS_UTL_safeWrite(F, &info, "Partition ovs file", sizeof(ovStoreInfo), 1); fclose(F); fprintf(stderr, "Wrote "F_U64" overlaps into '%s'\n", info._numOverlapsTotal, name); fprintf(stderr, " Smallest "F_U64"\n", info._smallestIID); fprintf(stderr, " Largest "F_U64"\n", info._largestIID); } // For the parallel sort, but also generally applicable, test that the index is sane. bool testIndex(char *ovlName, bool doFixes) { char name[FILENAME_MAX]; FILE *I = NULL; FILE *F = NULL; sprintf(name, "%s/index", ovlName); errno = 0; I = fopen(name, "r"); if (errno) fprintf(stderr, "ERROR: Failed to open '%s' for reading: %s\n", name, strerror(errno)), exit(1); //fprintf(stderr, "TESTING '%s'\n", name); if (doFixes) { sprintf(name, "%s/index.fixed", ovlName); errno = 0; F = fopen(name, "w"); if (errno) fprintf(stderr, "ERROR: Failed to open '%s' for writing: %s\n", name, strerror(errno)), exit(1); //fprintf(stderr, "WITH FIXES TO '%s'\n", name); } ovStoreOfft O; uint32 curIID = 0; uint32 minIID = UINT32_MAX; uint32 maxIID = 0; uint32 nErrs = 0; while (1 == AS_UTL_safeRead(I, &O, "offset", sizeof(ovStoreOfft), 1)) { bool maxIncreases = (maxIID < O._a_iid); bool errorDecreased = ((O._a_iid < curIID)); bool errorGap = ((O._a_iid > 0) && (curIID + 1 != O._a_iid)); if (O._a_iid < minIID) minIID = O._a_iid; if (maxIncreases) maxIID = O._a_iid; if (errorDecreased) fprintf(stderr, "ERROR: index decreased from "F_U32" to "F_U32"\n", curIID, O._a_iid), nErrs++; else if (errorGap) fprintf(stderr, "ERROR: gap between "F_U32" and "F_U32"\n", curIID, O._a_iid), nErrs++; if ((maxIncreases == true) && (errorGap == false)) { if (doFixes) AS_UTL_safeWrite(F, &O, "offset", sizeof(ovStoreOfft), 1); } else if (O._numOlaps > 0) { fprintf(stderr, "ERROR: lost overlaps a_iid "F_U32" fileno "F_U32" offset "F_U32" numOlaps "F_U32"\n", O._a_iid, O._fileno, O._offset, O._numOlaps); } curIID = O._a_iid; } fclose(I); if (F) fclose(F); return(nErrs == 0); } // For the parallel sort, merge index and info files into one, clean up the intermediates. void mergeInfoFiles(char *storePath, uint32 nPieces) { ovStoreInfo infopiece; ovStoreInfo info; info._ovsMagic = ovStoreMagic; info._ovsVersion = ovStoreVersion; info._smallestIID = UINT64_MAX; info._largestIID = 0; info._numOverlapsTotal = 0; info._highestFileIndex = nPieces; info._maxReadLenInBits = AS_MAX_READLEN_BITS; ovStoreOfft offm; offm._a_iid = 0; offm._fileno = 1; offm._offset = 0; offm._numOlaps = 0; offm._overlapID = 0; // Open the new master index output file char name[FILENAME_MAX]; sprintf(name, "%s/index", storePath); errno = 0; FILE *idx = fopen(name, "w"); if (errno) fprintf(stderr, "ERROR: Failed to open '%s': %s\n", name, strerror(errno)), exit(1); // Special case, we need an empty index for the zeroth fragment. AS_UTL_safeWrite(idx, &offm, "ovStore::mergeInfoFiles::offsetZero", sizeof(ovStoreOfft), 1); // Sanity checking, compare the number of overlaps processed against the overlapID // of each ovStoreOfft. uint64 totalOverlaps = 0; // Process each for (uint32 i=1; i<=nPieces; i++) { sprintf(name, "%s/%04d.info", storePath, i); fprintf(stderr, "Processing '%s'\n", name); if (AS_UTL_fileExists(name, FALSE, FALSE) == false) { fprintf(stderr, "ERROR: file '%s' not found.\n", name); exit(1); } { errno = 0; FILE *F = fopen(name, "r"); if (errno) fprintf(stderr, "ERROR: Failed to open '%s': %s\n", name, strerror(errno)), exit(1); AS_UTL_safeRead(F, &infopiece, "ovStore::mergeInfoFiles::infopiece", sizeof(ovStoreInfo), 1); fclose(F); } // Add empty index elements for missing overlaps if (infopiece._numOverlapsTotal == 0) { fprintf(stderr, " No overlaps found.\n"); continue; } assert(infopiece._smallestIID <= infopiece._largestIID); if (info._largestIID + 1 < infopiece._smallestIID) fprintf(stderr, " Adding empty records for fragments "F_U64" to "F_U64"\n", info._largestIID + 1, infopiece._smallestIID - 1); while (info._largestIID + 1 < infopiece._smallestIID) { offm._a_iid = info._largestIID + 1; //offm._fileno = set below, where the recs are written to the master file //offm._offset = set below, where the recs are written to the master file AS_UTL_safeWrite(idx, &offm, "ovStore::mergeInfoFiles::offsets", sizeof(ovStoreOfft), 1); info._largestIID++; } // Copy index elements for existing overlaps. While copying, update the supposed position // of any fragments with no overlaps. Without doing this, accessing the store beginning // or ending at such a fragment will fail. { sprintf(name, "%s/%04d.index", storePath, i); errno = 0; FILE *F = fopen(name, "r"); if (errno) fprintf(stderr, "ERROR: Failed to open '%s': %s\n", name, strerror(errno)), exit(1); uint32 recsLen = 0; uint32 recsMax = 1024 * 1024; ovStoreOfft *recs = new ovStoreOfft [recsMax]; recsLen = AS_UTL_safeRead(F, recs, "ovStore::mergeInfoFiles::offsetsLoad", sizeof(ovStoreOfft), recsMax); if (recsLen > 0) { if (info._largestIID + 1 != recs[0]._a_iid) fprintf(stderr, "ERROR: '%s' starts with iid "F_U32", but store only up to "F_U64"\n", name, recs[0]._a_iid, info._largestIID); assert(info._largestIID + 1 == recs[0]._a_iid); } while (recsLen > 0) { // Update location of missing reads. offm._fileno = recs[recsLen-1]._fileno; offm._offset = recs[recsLen-1]._offset; // Update overlapID for each record. for (uint32 rr=0; rr 0) assert(recs[rr]._overlapID == totalOverlaps); totalOverlaps += recs[rr]._numOlaps; } // Write the records, read next batch AS_UTL_safeWrite(idx, recs, "ovStore::mergeInfoFiles::offsetsWrite", sizeof(ovStoreOfft), recsLen); recsLen = AS_UTL_safeRead(F, recs, "ovStore::mergeInfoFiles::offsetsReLoad", sizeof(ovStoreOfft), recsMax); } delete [] recs; fclose(F); } // Update the info block to include the overlaps we just added info._smallestIID = MIN(info._smallestIID, infopiece._smallestIID); info._largestIID = MAX(info._largestIID, infopiece._largestIID); info._numOverlapsTotal += infopiece._numOverlapsTotal; fprintf(stderr, " Now finished with fragments "F_U64" to "F_U64" -- "F_U64" overlaps.\n", info._smallestIID, info._largestIID, info._numOverlapsTotal); } fclose(idx); // Dump the new store info file { sprintf(name, "%s/info", storePath); errno = 0; FILE *F = fopen(name, "w"); if (errno) fprintf(stderr, "ERROR: Failed to open '%s': %s\n", name, strerror(errno)), exit(1); AS_UTL_safeWrite(F, &info, "ovStore::mergeInfoFiles::finalInfo", sizeof(ovStoreInfo), 1); fclose(F); } fprintf(stderr, "\n"); fprintf(stderr, "Index finalized for reads "F_U64" to "F_U64" with "F_U64" overlaps.\n", info._smallestIID, info._largestIID, info._numOverlapsTotal); } // // // For overlap store building, both sequential and parallel. Overlap filtering. // // #define OBT_FAR5PRIME (29) #define OBT_MIN_LENGTH (75) // Are the 5' end points very different? If the overlap is flipped, then, yes, they are. static bool isOverlapDifferent(ovOverlap &ol) { bool isDiff = true; if (ol.flipped() == false) { if (ol.a_bgn() > ol.b_bgn()) isDiff = ((ol.a_bgn() - ol.b_bgn()) > OBT_FAR5PRIME) ? (true) : (false); else isDiff = ((ol.b_bgn() - ol.a_bgn()) > OBT_FAR5PRIME) ? (true) : (false); } return(isDiff); } // Is the overlap long? static bool isOverlapLong(ovOverlap &ol) { int32 ab = ol.a_bgn(); int32 ae = ol.a_end(); int32 bb = ol.b_bgn(); int32 be = ol.b_end(); int32 Alength = ae - ab; int32 Blength = be - bb; if (be < bb) Blength = bb - be; return(((Alength > OBT_MIN_LENGTH) && (Blength > OBT_MIN_LENGTH)) ? (true) : (false)); } void ovStoreFilter::filterOverlap(ovOverlap &foverlap, ovOverlap &roverlap) { // Quick sanity check on IIDs. if ((foverlap.a_iid == 0) || (foverlap.b_iid == 0) || (foverlap.a_iid >= maxID) || (foverlap.b_iid >= maxID)) { char ovlstr[256]; fprintf(stderr, "Overlap has IDs out of range (maxID "F_U32"), possibly corrupt input data.\n", maxID); fprintf(stderr, " coords -- %s\n", foverlap.toString(ovlstr, ovOverlapAsCoords, false)); fprintf(stderr, " hangs -- %s\n", foverlap.toString(ovlstr, ovOverlapAsHangs, false)); exit(1); } // Make the reverse overlap (important, AFTER resetting the erate-based 'for' flags). roverlap.swapIDs(foverlap); // Ignore high error overlaps if ((foverlap.evalue() > maxEvalue)) { foverlap.dat.ovl.forUTG = false; foverlap.dat.ovl.forOBT = false; foverlap.dat.ovl.forDUP = false; roverlap.dat.ovl.forUTG = false; roverlap.dat.ovl.forOBT = false; roverlap.dat.ovl.forDUP = false; skipERATE++; skipERATE++; } // Don't OBT if not requested. if ((foverlap.dat.ovl.forOBT == false) && (skipReadOBT[foverlap.a_iid] == true)) { foverlap.dat.ovl.forOBT = false; skipOBT++; } if ((roverlap.dat.ovl.forOBT == false) && (skipReadOBT[roverlap.a_iid] == true)) { roverlap.dat.ovl.forOBT = false; skipOBT++; } // If either overlap is good for either obt or dup, compute if it is different and long. These // are the same for both foverlap and roverlap. bool isDiff = isOverlapDifferent(foverlap); bool isLong = isOverlapLong(foverlap); // Remove the bad-for-OBT overlaps. if ((isDiff == false) && (foverlap.dat.ovl.forOBT == true)) { foverlap.dat.ovl.forOBT = false; skipOBTbad++; } if ((isDiff == false) && (roverlap.dat.ovl.forOBT == true)) { roverlap.dat.ovl.forOBT = false; skipOBTbad++; } // Remove the too-short-for-OBT overlaps. if ((isLong == false) && (foverlap.dat.ovl.forOBT == true)) { foverlap.dat.ovl.forOBT = false; skipOBTshort++; } if ((isLong == false) && (roverlap.dat.ovl.forOBT == true)) { roverlap.dat.ovl.forOBT = false; skipOBTshort++; } // Don't dedupe if not requested. if ((foverlap.dat.ovl.forDUP == true) && (skipReadDUP[foverlap.a_iid] == true)) { foverlap.dat.ovl.forDUP = false; skipDUP++; } if ((roverlap.dat.ovl.forDUP == true) && (skipReadDUP[roverlap.b_iid] == true)) { roverlap.dat.ovl.forDUP = false; skipDUP++; } // Remove the bad-for-DUP overlaps. #if 0 // Nah, do this in dedupe, since parameters can change. if ((isDiff == true) && (foverlap.dat.ovl.forDUP == true)) { foverlap.dat.ovl.forDUP = false; skipDUPdiff++; } if ((isDiff == true) && (roverlap.dat.ovl.forDUP == true)) { roverlap.dat.ovl.forDUP = false; skipDUPdiff++; } #endif // Can't have duplicates between libraries. if (((foverlap.dat.ovl.forDUP == true) || (roverlap.dat.ovl.forDUP == true)) && (gkp->gkStore_getRead(foverlap.a_iid)->gkRead_libraryID() != gkp->gkStore_getRead(foverlap.b_iid)->gkRead_libraryID())) { if ((foverlap.dat.ovl.forDUP == true)) { foverlap.dat.ovl.forDUP = false; skipDUPlib++; } if ((roverlap.dat.ovl.forDUP == true)) { roverlap.dat.ovl.forDUP = false; skipDUPlib++; } } // All done with the filtering, record some counts. if (foverlap.dat.ovl.forUTG == true) saveUTG++; if (foverlap.dat.ovl.forOBT == true) saveOBT++; if (foverlap.dat.ovl.forDUP == true) saveDUP++; if (roverlap.dat.ovl.forUTG == true) saveUTG++; if (roverlap.dat.ovl.forOBT == true) saveOBT++; if (roverlap.dat.ovl.forDUP == true) saveDUP++; } void ovStoreFilter::reportFate(void) { fprintf(stderr, "overlap fate:\n"); fprintf(stderr, "%16"F_U64P" SAVE - overlaps output (for unitigging)\n", saveUTG); fprintf(stderr, "%16"F_U64P" SAVE - overlaps output (for OBT)\n", saveOBT); fprintf(stderr, "%16"F_U64P" SAVE - overlaps output (for dedupe)\n", saveDUP); fprintf(stderr, "\n"); fprintf(stderr, "%16"F_U64P" ERATE - low quality, more than %.3f fraction error\n", skipERATE, AS_OVS_decodeEvalue(maxEvalue)); fprintf(stderr, "\n"); fprintf(stderr, "%16"F_U64P" OBT - not requested\n", skipOBT); fprintf(stderr, "%16"F_U64P" OBT - too similar\n", skipOBTbad); fprintf(stderr, "%16"F_U64P" OBT - too short\n", skipOBTshort); fprintf(stderr, "\n"); fprintf(stderr, "%16"F_U64P" DUP - dedupe not requested\n", skipDUP); fprintf(stderr, "%16"F_U64P" DUP - different library\n", skipDUPlib); fprintf(stderr, "%16"F_U64P" DUP - obviously not duplicates\n", skipDUPdiff); } void ovStoreFilter::resetCounters(void) { saveUTG = 0; saveOBT = 0; saveDUP = 0; skipERATE = 0; skipOBT = 0; skipOBTbad = 0; skipOBTshort = 0; skipDUP = 0; skipDUPdiff = 0; skipDUPlib = 0; } canu-1.3/src/stores/ovStore.H000066400000000000000000000555101272335422300161740ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2014-DEC-09 to 2015-JUL-01 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-12 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-MAR-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef AS_OVERLAP_H #define AS_OVERLAP_H #include "AS_global.H" #include "gkStore.H" // Error rates are encoded as a 12-bit fixed-point value. This gives us up to 40.95% error, with // 0.01% resolution. Changing the number of bits WILL break the carefully structured // ovOverlapDAT. // // The decoded value is a double representing fraction error -- between 0.0000 and 1.0000. // The encoded value is an integer type (see the ovsOvelrapDAT below). #define AS_MAX_EVALUE_BITS 12 #define AS_MAX_EVALUE ((1 << AS_MAX_EVALUE_BITS) - 1) #define AS_OVS_decodeEvalue(E) ((E) / 10000.0) #define AS_OVS_encodeEvalue(Q) (((Q) < AS_OVS_decodeEvalue(AS_MAX_EVALUE)) ? (int)(10000.0 * (Q) + 0.5) : AS_MAX_EVALUE) #define AS_MAX_ERATE AS_OVS_decodeEvalue(AS_MAX_EVALUE) // The old implementation allowed up to 20-bit reads, and used 3 32-bit words. No alignment was // stored. // // The new implementation uses either 5 (for EXACTLY 16-bit reads) or 6 32-bit words. It uses 2 // 32-bit words for storing a pointer to the alignments. // // Note that the 5-word version must use uint32, and the 6-word version must use uint64. // Note that the 5-word version needs to split out the alignPos into two words. #if AS_MAX_READLEN_BITS < 17 #define ovOverlapNWORDS 5 typedef uint32 ovOverlapWORD; #define F_OV F_U32 #define F_OVP F_U32P class ovOverlapDAT { public: ovOverlapWORD ahg5 : AS_MAX_READLEN_BITS; // 16 ovOverlapWORD ahg3 : AS_MAX_READLEN_BITS; // 16 ovOverlapWORD bhg5 : AS_MAX_READLEN_BITS; // 16 ovOverlapWORD bhg3 : AS_MAX_READLEN_BITS; // 16 ovOverlapWORD span : AS_MAX_READLEN_BITS; // 16 ovOverlapWORD evalue : AS_MAX_EVALUE_BITS; // 12 ovOverlapWORD flipped : 1; // 1 ovOverlapWORD forOBT : 1; // 1 ovOverlapWORD forDUP : 1; // 1 ovOverlapWORD forUTG : 1; // 1 ovOverlapWORD alignSwapped : 1; // Our IDs are opposite those in the alignment ovOverlapWORD alignFile : 19; // Which file of overlap alignments ovOverlapWORD alignPosHi : 12; // Position in that file (high-order bits) ovOverlapWORD alignPosLo : 32; // Position in that file (low-order bits) }; #elif AS_MAX_READLEN_BITS < 22 #define ovOverlapNWORDS 3 typedef uint64 ovOverlapWORD; #define F_OV F_U64 #define F_OVP F_U64P class ovOverlapDAT { public: ovOverlapWORD ahg5 : AS_MAX_READLEN_BITS; // 17-21 ovOverlapWORD ahg3 : AS_MAX_READLEN_BITS; // 17-21 ovOverlapWORD evalue : AS_MAX_EVALUE_BITS; // 12 ovOverlapWORD flipped : 1; // 1 ovOverlapWORD forOBT : 1; // 1 ovOverlapWORD forDUP : 1; // 1 ovOverlapWORD forUTG : 1; // 1 ovOverlapWORD extra1 : 64 - 2 * AS_MAX_READLEN_BITS - AS_MAX_EVALUE_BITS - 1 - 1 - 1 - 1; // Between 15 and 7 ovOverlapWORD bhg5 : AS_MAX_READLEN_BITS; // 17-21 ovOverlapWORD bhg3 : AS_MAX_READLEN_BITS; // 17-21 ovOverlapWORD span : AS_MAX_READLEN_BITS; // 17-21 ovOverlapWORD extra2 : 64 - 3 * AS_MAX_READLEN_BITS; // Between 13 and 1 ovOverlapWORD alignSwapped : 1; // Our IDs are opposite those in the alignment ovOverlapWORD alignFile : 19; // Which file of overlap alignments ovOverlapWORD alignPos : 44; // Position in that file }; #else #define ovOverlapNWORDS 8 typedef uint32 ovOverlapWORD; #define F_OV F_U32 #define F_OVP F_U32P class ovOverlapDAT { public: ovOverlapWORD ahg5; ovOverlapWORD ahg3; ovOverlapWORD bhg5; ovOverlapWORD bhg3; ovOverlapWORD span; ovOverlapWORD evalue : AS_MAX_EVALUE_BITS; // 12 ovOverlapWORD flipped : 1; // 1 ovOverlapWORD forOBT : 1; // 1 ovOverlapWORD forDUP : 1; // 1 ovOverlapWORD forUTG : 1; // 1 ovOverlapWORD extra : 32 - AS_MAX_EVALUE_BITS - 1 - 1 - 1 - 1; // Between 15 and 7 ovOverlapWORD alignSwapped : 1; // Our IDs are opposite those in the alignment ovOverlapWORD alignFile : 19; // Which file of overlap alignments ovOverlapWORD alignPosHi : 12; // Position in that file (high-order bits) ovOverlapWORD alignPosLo : 32; // Position in that file (low-order bits) }; #endif enum ovOverlapDisplayType { ovOverlapAsHangs = 0, // Show a and b hang ovOverlapAsCoords = 1, // Show bgn,end for each read ovOverlapAsRaw = 2, // Show all four hangs ovOverlapAsCompat = 3, // Show in a format more-or-less compatible with CA 8.3 ovOverlapAsPaf = 4, // Show in a format compatible with miniasm }; class ovOverlap { private: ovOverlap() { clear(); }; public: ovOverlap(gkStore *gkp) { g = gkp; clear(); }; ~ovOverlap() { }; static ovOverlap *allocateOverlaps(gkStore *gkp, uint64 num) { ovOverlap *r = new ovOverlap [num]; for (uint32 ii=0; ii 0 ---> ahg5 > 0 (and bhg5 == 0) // a_hang < 0 ---> bhg5 > 0 (and ahg5 == 0) // // b_hang > 0 ---> bhg3 > 0 (and ahg3 == 0) // b_hang < 0 ---> ahg3 > 0 (and bhg3 == 0) // // Convenience functions. int32 a_hang(void) const { return((int32)dat.ovl.ahg5 - (int32)dat.ovl.bhg5); }; int32 b_hang(void) const { return((int32)dat.ovl.bhg3 - (int32)dat.ovl.ahg3); }; void a_hang(int32 a) { dat.ovl.ahg5 = (a < 0) ? 0 : a; dat.ovl.bhg5 = (a < 0) ? -a : 0; }; void b_hang(int32 b) { dat.ovl.bhg3 = (b < 0) ? 0 : b; dat.ovl.ahg3 = (b < 0) ? -b : 0; }; // These return the actual coordinates on the read. For reverse B reads, the coordinates are in the reverse-complemented // sequence, and are returned as bgn > end to show this. uint32 a_bgn(void) const { return(dat.ovl.ahg5); }; uint32 a_end(void) const { return(g->gkStore_getRead(a_iid)->gkRead_sequenceLength() - dat.ovl.ahg3); }; uint32 b_bgn(void) const { return((dat.ovl.flipped) ? (g->gkStore_getRead(b_iid)->gkRead_sequenceLength() - dat.ovl.bhg5) : (dat.ovl.bhg5)); }; uint32 b_end(void) const { return((dat.ovl.flipped) ? (dat.ovl.bhg3) : (g->gkStore_getRead(b_iid)->gkRead_sequenceLength() - dat.ovl.bhg3)); }; uint32 span(void) const { return(dat.ovl.span); }; void span(uint32 s) { dat.ovl.span = s; }; #if 0 // Return an approximate span as the average of the read span aligned. uint32 span(void) const { if (dat.ovl.span > 0) return(dat.ovl.span); else { uint32 ab = a_bgn(), ae = a_end(); uint32 bb = b_bgn(), be = b_end(); if (bb < be) return(((ae - ab) + (be - bb)) / 2); else return(((ae - ab) + (bb - be)) / 2); } } #endif void flipped(uint32 f) { dat.ovl.flipped = f; }; uint32 flipped(void) const { return(dat.ovl.flipped == true); }; void erate(double e) { dat.ovl.evalue = AS_OVS_encodeEvalue(e); }; double erate(void) const { return(AS_OVS_decodeEvalue(dat.ovl.evalue)); }; void evalue(uint64 e) { dat.ovl.evalue = e; }; uint64 evalue(void) const { return(dat.ovl.evalue); }; bool forOBT(void) { return(dat.ovl.forOBT); }; bool forDUP(void) { return(dat.ovl.forDUP); }; bool forUTG(void) { return(dat.ovl.forUTG); }; // These are true only if the overlap is dovetail, which is the usual case, and isn't checked. uint32 overlapAEndIs5prime(void) const { return((dat.ovl.bhg5 > 0) && (dat.ovl.ahg3 > 0)); }; uint32 overlapAEndIs3prime(void) const { return((dat.ovl.ahg5 > 0) && (dat.ovl.bhg3 > 0)); }; uint32 overlapBEndIs5prime(void) const { return((overlapAEndIs5prime() && (dat.ovl.flipped == true)) || (overlapAEndIs3prime() && (dat.ovl.flipped == false))); }; uint32 overlapBEndIs3prime(void) const { return((overlapAEndIs5prime() && (dat.ovl.flipped == false)) || (overlapAEndIs3prime() && (dat.ovl.flipped == true))); }; uint32 overlapAIsContained(void) const { return((dat.ovl.ahg5 == 0) && (dat.ovl.ahg3 == 0)); }; uint32 overlapBIsContainer(void) const { return((dat.ovl.ahg5 == 0) && (dat.ovl.ahg3 == 0)); }; uint32 overlapAIsContainer(void) const { return((dat.ovl.bhg5 == 0) && (dat.ovl.bhg3 == 0)); }; uint32 overlapBIsContained(void) const { return((dat.ovl.bhg5 == 0) && (dat.ovl.bhg3 == 0)); }; // Test if the overlap is dovetail or partial. uint32 overlap5primeIsPartial(void) const { return((dat.ovl.ahg5 > 0) && (dat.ovl.bhg5 > 0)); }; uint32 overlap3primeIsPartial(void) const { return((dat.ovl.ahg3 > 0) && (dat.ovl.bhg3 > 0)); }; uint32 overlapIsPartial(void) const { return(overlap5primeIsPartial() || overlap3primeIsPartial()); }; char *toString(char *str, ovOverlapDisplayType type, bool newLine); void swapIDs(ovOverlap const &orig); void clear(void) { dat.dat[0] = 0; dat.dat[1] = 0; dat.dat[2] = 0; #if (ovOverlapNWORDS == 5) dat.dat[3] = 0; dat.dat[4] = 0; #endif }; bool operator<(const ovOverlap &that) const { if (a_iid < that.a_iid) return(true); if (a_iid > that.a_iid) return(false); if (b_iid < that.b_iid) return(true); if (b_iid > that.b_iid) return(false); if (dat.dat[0] < that.dat.dat[0]) return(true); if (dat.dat[0] > that.dat.dat[0]) return(false); if (dat.dat[1] < that.dat.dat[1]) return(true); if (dat.dat[1] > that.dat.dat[1]) return(false); if (dat.dat[2] < that.dat.dat[2]) return(true); if (dat.dat[2] > that.dat.dat[2]) return(false); #if (ovOverlapNWORDS == 5) if (dat.dat[3] < that.dat.dat[3]) return(true); if (dat.dat[3] > that.dat.dat[3]) return(false); if (dat.dat[4] < that.dat.dat[4]) return(true); if (dat.dat[4] > that.dat.dat[4]) return(false); #endif return(false); }; public: gkStore *g; public: uint32 a_iid; uint32 b_iid; union { ovOverlapWORD dat[ovOverlapNWORDS]; ovOverlapDAT ovl; } dat; }; // The default, no flags, is to open for normal overlaps, read only. Normal overlaps mean they // have only the B id, i.e., they are in a fully built store. // // Output of overlapper (input to store building) should be ovFileFullWrite. The specialized // ovFileFullWriteNoCounts is used internally by store creation. // enum ovFileType { ovFileNormal = 0, // Reading of b_id overlaps (aka store files) ovFileNormalWrite = 1, // Writing of b_id overlaps ovFileFull = 2, // Reading of a_id+b_id overlaps (aka dump files) ovFileFullWrite = 3, // Writing of a_id+b_id overlaps ovFileFullWriteNoCounts = 4 // Writing of a_id+b_id overlaps, omitting the counts of olaps per read }; class ovFile { public: ovFile(const char *name, ovFileType type = ovFileNormal, uint32 bufferSize = 1 * 1024 * 1024); ~ovFile(); void flushOverlaps(void); void writeOverlap(ovOverlap *overlap); void writeOverlaps(ovOverlap *overlaps, uint64 overlapLen); bool readOverlap(ovOverlap *overlap); uint64 readOverlaps(ovOverlap *overlaps, uint64 overlapMax); void seekOverlap(off_t overlap); // The size of an overlap record is 1 or 2 IDs + the size of a word times the number of words. uint64 recordSize(void) { return(sizeof(uint32) * ((_isNormal) ? 1 : 2) + sizeof(ovOverlapWORD) * ovOverlapNWORDS); }; private: uint32 _bufferLen; // length of valid data in the buffer uint32 _bufferPos; // position the read is at in the buffer uint32 _bufferMax; // allocated size of the buffer uint32 *_buffer; uint32 _olapsPerReadAlloc; uint32 _olapsPerReadLast; uint32 *_olapsPerRead; bool _isOutput; // if true, we can writeOverlap() bool _isSeekable; // if true, we can seekOverlap() bool _isNormal; // if true, 3 words per overlap, else 4 compressedFileReader *_reader; compressedFileWriter *_writer; char _prefix[FILENAME_MAX]; FILE *_file; }; class ovStoreInfo { private: uint64 _ovsMagic; uint64 _ovsVersion; uint64 _UNUSED; uint64 _smallestIID; // smallest frag iid in the store uint64 _largestIID; // largest frag iid in the store uint64 _numOverlapsTotal; // number of overlaps in the store uint64 _highestFileIndex; uint64 _maxReadLenInBits; // length of a fragment friend class ovStore; friend void writeOverlaps(char *storePath, ovOverlap *ovls, uint64 ovlsLen, uint32 fileID); friend bool testIndex(char *storePath, bool doFixes); friend void mergeInfoFiles(char *storePath, uint32 nPieces); }; class ovStoreOfft { private: uint32 _a_iid; // read ID for this block of overlaps. uint32 _fileno; // the file that contains this a_iid uint32 _offset; // offset to the first overlap for this iid uint32 _numOlaps; // number of overlaps for this iid uint64 _overlapID; // overlapID for the first overlap in this block. in memory, this is the id of the next overlap. void clear(void) { _a_iid = 0; _fileno = 0; _offset = 0; _numOlaps = 0; _overlapID = 0; }; friend class ovStore; friend void writeOverlaps(char *storePath, ovOverlap *ovls, uint64 ovlsLen, uint32 fileID); friend bool testIndex(char *storePath, bool doFixes); friend void mergeInfoFiles(char *storePath, uint32 nPieces); }; // The default here is to open a read only store. // enum ovStoreType { ovStoreReadOnly = 0, ovStoreWrite = 1, // Open for write, fail if one exists already ovStoreOverwrite = 2, // Open for write, and obliterate an existing store }; class ovStore { private: void ovStore_read(void); void ovStore_write(void); public: ovStore(const char *name, gkStore *gkp, ovStoreType cType=ovStoreReadOnly); ~ovStore(); // Read the next overlap from the store. Return value is the number of overlaps read. uint32 readOverlap(ovOverlap *overlap); // Return the number of overlaps that would be read. Basically the same as the next readOverlaps() call. uint32 numberOfOverlaps(void); // Read ALL remaining overlaps for the current A_iid. Return value is the number of overlaps read. uint32 readOverlaps(ovOverlap *&overlaps, uint32 &maxOverlaps, bool restrictToIID=true); // Append ALL remaining overlaps for the current A_iid to the overlaps in ovl. Return value is // the number of overlaps in ovl that are for A_iid == iid. // // It is up to the client to verify that ovl[0] is the same as iid (e.g., that the return value // is not zero); ovlLen is the number of overlaps in ovl, NOT the number of overlaps in ovl that // are the same as iid. // uint32 readOverlaps(uint32 iid, ovOverlap *&ovl, uint32 &ovlLen, uint32 &ovlMax); void setRange(uint32 low, uint32 high); void resetRange(void); uint64 numOverlapsInRange(void); uint32 * numOverlapsPerFrag(uint32 &firstFrag, uint32 &lastFrag); // The (mostly) private interface for adding overlaps to a store. Overlaps must be sorted already. void writeOverlap(ovOverlap *olap); void writeOverlap(ovOverlap *overlap, uint32 maxOverlapsThisFile); // Write a block of sorted overlaps to store file 'fileID', saving the info and index into // 'fileID.info' and 'fileID.index' friend void writeOverlaps(char *storePath, ovOverlap *ovls, uint64 ovlsLen, uint32 fileID); // Add new evalues for reads between bgnID and endID. No checking of IDs is done, but the number // of evalues must agree. void addEvalues(uint32 bgnID, uint32 endID, uint16 *evalues, uint64 evaluesLen); private: char _storePath[FILENAME_MAX]; bool _isOutput; ovStoreInfo _info; uint32 _firstIIDrequested; uint32 _lastIIDrequested; FILE *_offtFile; // For writing overlaps, a place to dump ovStoreOfft's. ovStoreOfft _offt; // For writing overlaps, the current ovStoreOfft. ovStoreOfft _offm; // For writing overlaps, an empty ovStoreOfft, for reads with no overlaps. memoryMappedFile *_evaluesMap; uint16 *_evalues; uint64 _overlapsThisFile; // Count of the number of overlaps written so far uint32 _currentFileIndex; ovFile *_bof; gkStore *_gkp; }; // This should be part of ovStore, but when it is used, in ovStoreSorter, we don't // have a store opened. void writeOverlaps(char *storePath, ovOverlap *ovls, uint64 ovlsLen, uint32 fileID); bool testIndex(char *storePath, bool doFixes); void mergeInfoFiles(char *storePath, uint32 nPieces); // For store construction. Probably should be in either ovOverlap or ovStore. class ovStoreFilter { public: ovStoreFilter(gkStore *gkp_, double maxErate) { gkp = gkp_; resetCounters(); maxID = gkp->gkStore_getNumReads() + 1; maxEvalue = AS_OVS_encodeEvalue(maxErate); skipReadOBT = new char [maxID]; skipReadDUP = new char [maxID]; memset(skipReadOBT, 0, sizeof(char) * maxID); memset(skipReadDUP, 0, sizeof(char) * maxID); uint32 numSkipOBT = 0; uint32 numSkipDUP = 0; fprintf(stderr, "Marking fragments to skip overlap based trimming.\n"); fprintf(stderr, "LIB 1 - dup=%d trim=%d spur=%d chimera=%d subreads=%d\n", gkp->gkStore_getLibrary(1)->gkLibrary_removeDuplicateReads(), gkp->gkStore_getLibrary(1)->gkLibrary_finalTrim(), gkp->gkStore_getLibrary(1)->gkLibrary_removeSpurReads(), gkp->gkStore_getLibrary(1)->gkLibrary_removeChimericReads(), gkp->gkStore_getLibrary(1)->gkLibrary_checkForSubReads()); for (uint64 iid=0; iidgkStore_getRead(iid)->gkRead_libraryID(); gkLibrary *L = gkp->gkStore_getLibrary(Lid); if ((L->gkLibrary_removeDuplicateReads() == false) && (L->gkLibrary_finalTrim() == GK_FINALTRIM_NONE) && (L->gkLibrary_removeSpurReads() == false) && (L->gkLibrary_removeChimericReads() == false) && (L->gkLibrary_checkForSubReads() == false)) { numSkipOBT++; skipReadOBT[iid] = true; } if (L->gkLibrary_removeDuplicateReads() == false) { numSkipDUP++; skipReadDUP[iid] = true; } } fprintf(stderr, "Marked "F_U32" reads so skip OBT, "F_U32" reads to skip dedupe.\n", numSkipOBT, numSkipDUP); }; ~ovStoreFilter() { delete [] skipReadOBT; delete [] skipReadDUP; }; void filterOverlap(ovOverlap &foverlap, ovOverlap &roverlap); void reportFate(void); void resetCounters(void); public: gkStore *gkp; uint32 maxID; uint32 maxEvalue; uint64 saveUTG; uint64 saveOBT; uint64 saveDUP; uint64 skipERATE; uint64 skipOBT; // OBT not requested for the A read uint64 skipOBTbad; // Overlap too similiar uint64 skipOBTshort; // Overlap is too short uint64 skipDUP; // DUP not requested for the A read uint64 skipDUPdiff; // Overlap isn't remotely similar uint64 skipDUPlib; // Not really stats, but global state for the filter. char *skipReadOBT; char *skipReadDUP; }; #endif canu-1.3/src/stores/ovStoreBucketizer.C000066400000000000000000000255141272335422300202200ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVS/overlapStoreBucketizer.C * * Modifications by: * * Brian P. Walenz from 2012-APR-02 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-AUG-22 to 2015-SEP-21 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-08 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-FEB-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" static void writeToFile(ovOverlap *overlap, ovFile **sliceFile, uint32 sliceFileMax, uint64 *sliceSize, uint32 *iidToBucket, char *ovlName, uint32 jobIndex, bool useGzip) { uint32 df = iidToBucket[overlap->a_iid]; if (sliceFile[df] == NULL) { char name[FILENAME_MAX]; sprintf(name, "%s/create%04d/slice%03d%s", ovlName, jobIndex, df, (useGzip) ? ".gz" : ""); sliceFile[df] = new ovFile(name, ovFileFullWriteNoCounts); sliceSize[df] = 0; } sliceFile[df]->writeOverlap(overlap); sliceSize[df]++; } // These are duplicated between ovStoreBucketizer and ovStoreBuild static void markOBT(gkStore *gkp, uint32 maxIID, char *skipRead) { uint64 numMarked = 0; if (skipRead == NULL) return; fprintf(stderr, "Marking fragments to skip overlap based trimming.\n"); for (uint64 iid=0; iidgkStore_getRead(iid)->gkRead_libraryID(); gkLibrary *L = gkp->gkStore_getLibrary(Lid); if (L == NULL) continue; if ((L->gkLibrary_removeDuplicateReads() == false) && (L->gkLibrary_finalTrim() != GK_FINALTRIM_LARGEST_COVERED) && (L->gkLibrary_removeSpurReads() == false) && (L->gkLibrary_removeChimericReads() == false)) { numMarked++; skipRead[iid] = true; } } fprintf(stderr, "Marked "F_U64" fragments.\n", numMarked); } static void markDUP(gkStore *gkp, uint32 maxIID, char *skipRead) { uint64 numMarked = 0; if (skipRead == NULL) return; fprintf(stderr, "Marking fragments to skip deduplication.\n"); for (uint64 iid=0; iidgkStore_getRead(iid)->gkRead_libraryID(); gkLibrary *L = gkp->gkStore_getLibrary(Lid); if (L == NULL) continue; if (L->gkLibrary_removeDuplicateReads() == false) { numMarked++; skipRead[iid] = true; } } fprintf(stderr, "Marked "F_U64" fragments.\n", numMarked); } int main(int argc, char **argv) { char *ovlName = NULL; char *gkpName = NULL; char *cfgName = NULL; uint32 maxFiles = sysconf(_SC_OPEN_MAX) - 16; uint32 fileLimit = maxFiles; uint32 jobIndex = 0; double maxErrorRate = 1.0; uint64 maxError = AS_OVS_encodeEvalue(maxErrorRate); char *ovlInput = NULL; bool useGzip = true; argc = AS_configure(argc, argv); int err=0; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-O") == 0) { ovlName = argv[++arg]; } else if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-C") == 0) { cfgName = argv[++arg]; } else if (strcmp(argv[arg], "-F") == 0) { fileLimit = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-job") == 0) { jobIndex = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-i") == 0) { ovlInput = argv[++arg]; } else if (strcmp(argv[arg], "-e") == 0) { maxErrorRate = atof(argv[++arg]); maxError = AS_OVS_encodeEvalue(maxErrorRate); } else if (strcmp(argv[arg], "-raw") == 0) { useGzip = false; } else { fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); } arg++; } if (ovlName == NULL) err++; if (gkpName == NULL) err++; if (ovlInput == NULL) err++; if (jobIndex == 0) err++; if (fileLimit > maxFiles) err++; if (err) { fprintf(stderr, "usage: %s -O asm.ovlStore -G asm.gkpStore -i file.ovb.gz -job j [opts]\n", argv[0]); fprintf(stderr, " -O asm.ovlStore path to store to create\n"); fprintf(stderr, " -G asm.gkpStore path to gkpStore for this assembly\n"); fprintf(stderr, "\n"); fprintf(stderr, " -C config path to previously created ovStoreBuild config data file\n"); fprintf(stderr, "\n"); fprintf(stderr, " -i file.ovb.gz input overlaps\n"); fprintf(stderr, " -job j index of this overlap input file\n"); fprintf(stderr, "\n"); fprintf(stderr, " -F f use up to 'f' files for store creation\n"); fprintf(stderr, "\n"); fprintf(stderr, " -obt filter overlaps for OBT\n"); fprintf(stderr, " -dup filter overlaps for OBT/dedupe\n"); fprintf(stderr, "\n"); fprintf(stderr, " -e e filter overlaps above e fraction error\n"); fprintf(stderr, "\n"); fprintf(stderr, " -raw write uncompressed buckets\n"); fprintf(stderr, "\n"); fprintf(stderr, " DANGER DO NOT USE DO NOT USE DO NOT USE DANGER\n"); fprintf(stderr, " DANGER DANGER\n"); fprintf(stderr, " DANGER This command is difficult to run by hand. DANGER\n"); fprintf(stderr, " DANGER Use ovStoreCreate instead. DANGER\n"); fprintf(stderr, " DANGER DANGER\n"); fprintf(stderr, " DANGER DO NOT USE DO NOT USE DO NOT USE DANGER\n"); fprintf(stderr, "\n"); if (ovlName == NULL) fprintf(stderr, "ERROR: No overlap store (-O) supplied.\n"); if (gkpName == NULL) fprintf(stderr, "ERROR: No gatekeeper store (-G) supplied.\n"); if (ovlInput == NULL) fprintf(stderr, "ERROR: No input (-i) supplied.\n"); if (jobIndex == 0) fprintf(stderr, "ERROR: No job index (-job) supplied.\n"); if (fileLimit > maxFiles) fprintf(stderr, "ERROR: Too many jobs (-F); only "F_SIZE_T" supported on this architecture.\n", maxFiles); exit(1); } { if (AS_UTL_fileExists(ovlName, TRUE, FALSE) == false) AS_UTL_mkdir(ovlName); } { char name[FILENAME_MAX]; sprintf(name, "%s/create%04d", ovlName, jobIndex); if (AS_UTL_fileExists(name, TRUE, FALSE) == false) AS_UTL_mkdir(name); else fprintf(stderr, "Overwriting previous result; directory '%s' exists.\n", name), exit(0); } { char name[FILENAME_MAX]; sprintf(name, "%s/bucket%04d/sliceSizes", ovlName, jobIndex); if (AS_UTL_fileExists(name, FALSE, FALSE) == true) fprintf(stderr, "Job finished; file '%s' exists.\n", name), exit(0); } gkStore *gkp = gkStore::gkStore_open(gkpName); uint32 maxIID = gkp->gkStore_getNumReads() + 1; uint32 *iidToBucket = new uint32 [maxIID]; { errno = 0; FILE *C = fopen(cfgName, "r"); if (errno) fprintf(stderr, "ERROR: failed to open config file '%s' for reading: %s\n", cfgName, strerror(errno)), exit(1); uint32 maxIIDtest = 0; AS_UTL_safeRead(C, &maxIIDtest, "maxIID", sizeof(uint32), 1); AS_UTL_safeRead(C, iidToBucket, "iidToBucket", sizeof(uint32), maxIID); if (maxIIDtest != maxIID) fprintf(stderr, "ERROR: maxIID in store ("F_U32") differs from maxIID in config file ("F_U32").\n", maxIID, maxIIDtest), exit(1); } ovFile **sliceFile = new ovFile * [fileLimit + 1]; uint64 *sliceSize = new uint64 [fileLimit + 1]; memset(sliceFile, 0, sizeof(ovFile *) * (fileLimit + 1)); memset(sliceSize, 0, sizeof(uint64) * (fileLimit + 1)); fprintf(stderr, "maxError fraction: %.3f percent: %.3f encoded: "F_U64"\n", maxErrorRate, maxErrorRate * 100, maxError); fprintf(stderr, "Bucketizing %s\n", ovlInput); ovStoreFilter *filter = new ovStoreFilter(gkp, maxError); ovOverlap foverlap(gkp); ovOverlap roverlap(gkp); ovFile *inputFile = new ovFile(ovlInput, ovFileFull); // Do bigger buffers increase performance? Do small ones hurt? //AS_OVS_setBinaryOverlapFileBufferSize(2 * 1024 * 1024); while (inputFile->readOverlap(&foverlap)) { filter->filterOverlap(foverlap, roverlap); // The filter copies f into r // If all are skipped, don't bother writing the overlap. if ((foverlap.dat.ovl.forUTG == true) || (foverlap.dat.ovl.forOBT == true) || (foverlap.dat.ovl.forDUP == true)) writeToFile(&foverlap, sliceFile, fileLimit, sliceSize, iidToBucket, ovlName, jobIndex, useGzip); if ((roverlap.dat.ovl.forUTG == true) || (roverlap.dat.ovl.forOBT == true) || (roverlap.dat.ovl.forDUP == true)) writeToFile(&roverlap, sliceFile, fileLimit, sliceSize, iidToBucket, ovlName, jobIndex, useGzip); } delete inputFile; filter->reportFate(); filter->resetCounters(); delete filter; for (uint32 i=0; i<=fileLimit; i++) delete sliceFile[i]; // Write slice sizes, rename bucket. { char name[FILENAME_MAX]; char finl[FILENAME_MAX]; sprintf(name, "%s/create%04d/sliceSizes", ovlName, jobIndex); FILE *F = fopen(name, "w"); if (errno) fprintf(stderr, "ERROR: Failed to open %s: %s\n", name, strerror(errno)), exit(1); AS_UTL_safeWrite(F, sliceSize, "sliceSize", sizeof(uint64), fileLimit + 1); fclose(F); sprintf(name, "%s/create%04d", ovlName, jobIndex); sprintf(finl, "%s/bucket%04d", ovlName, jobIndex); errno = 0; rename(name, finl); if (errno) fprintf(stderr, "ERROR: Failed to rename '%s' to final name '%s': %s\n", name, finl, strerror(errno)); } delete [] sliceFile; delete [] sliceSize; return(0); } canu-1.3/src/stores/ovStoreBucketizer.mk000066400000000000000000000007431272335422300204420ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := ovStoreBucketizer SOURCES := ovStoreBucketizer.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/ovStoreBuild.C000066400000000000000000000477361272335422300171620ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVS/overlapStoreBuild.C * * Modifications by: * * Brian P. Walenz from 2012-APR-02 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-JUL-31 to 2015-SEP-21 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-03 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-FEB-20 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "AS_UTL_decodeRange.H" #include "gkStore.H" #include "ovStore.H" #include #include using namespace std; #define MEMORY_OVERHEAD (256 * 1024 * 1024) // This is the size of the datastructure that we're using to store overlaps for sorting. // At present, with ovOverlap, it is over-allocating a pointer that we don't need, but // to make a custom structure, we'd need to duplicate a bunch of code or copy data after // loading and before writing. // // Used in both ovStoreSorter.C and ovStoreBuild.C. // #define ovOverlapSortSize (sizeof(ovOverlap)) static uint32 * computeIIDperBucket(uint32 fileLimit, uint64 minMemory, uint64 maxMemory, uint32 maxIID, vector &fileList) { uint32 *iidToBucket = new uint32 [maxIID]; uint32 maxFiles = MIN(floor(sysconf(_SC_CHILD_MAX) / 2), sysconf(_SC_OPEN_MAX) - 16); // If we're reading from stdin, not much we can do but divide the IIDs equally per file. Note // that the IIDs must be consecutive; the obvious, simple and clean division of 'mod' won't work. if (fileList[0][0] == '-') { if (maxMemory > 0) { minMemory = 0; maxMemory = 0; fileLimit = maxFiles; fprintf(stderr, "WARNING: memory limit (-M) specified, but can't be used with inputs from stdin; using %d files instead.\n", fileLimit); } else { fprintf(stderr, "Sorting overlaps from stdin using %d files.\n", fileLimit); } uint32 iidPerBucket = maxIID / fileLimit; uint32 thisBucket = 1; uint32 iidThisBucket = 0; for (uint32 ii=0; ii iidPerBucket) { iidThisBucket = 0; thisBucket++; } } return(iidToBucket); } // Otherwise, we have files, and should have counts. uint32 *overlapsPerRead = new uint32 [maxIID]; // Sum over all files. memset(overlapsPerRead, 0, sizeof(uint32) * maxIID); // For each overlap file, find the counts file and merge into overlapsPerRead. for (uint32 i=0; i 0) { olapsPerBucketMax = (uint64)ceil((double)numOverlaps / (double)fileLimit); fprintf(stderr, "Will sort using "F_U32" files; "F_U64" (%.2f million) overlaps per bucket; %.2f GB memory per bucket\n", fileLimit, olapsPerBucketMax, olapsPerBucketMax / 1000000.0, olapsPerBucketMax * GBperOlap); } // If a memory limit, distribute the overlaps to files no larger than the limit. // // This will pick the smallest memory size that uses fewer than maxFiles buckets. Unreasonable // values can break this - either too low memory or too high allowed open files (an OS limit). if (maxMemory > 0) { fprintf(stderr, "Configuring for %.2f GB to %.2f GB memory.\n", minMemory / 1024.0 / 1024.0 / 1024.0, maxMemory / 1024.0 / 1024.0 / 1024.0); if (minMemory < MEMORY_OVERHEAD + ovOverlapSortSize) minMemory = MEMORY_OVERHEAD + ovOverlapSortSize; uint64 incr = (maxMemory - minMemory) / 1000; if (incr < 1) incr = 1; // iterate until we can fit the files into file system limits. do { olapsPerBucketMax = (minMemory - MEMORY_OVERHEAD) / ovOverlapSortSize; minMemory += incr; } while ((minMemory <= maxMemory) && (numOverlaps / olapsPerBucketMax + 1 > 0.50 * maxFiles)); // Should we prefer finding 0.50 * maxFiles/2 (as above) but allow up to, say, 0.75 * maxFiles if 0.50 can't be satisfied? // Is the 0.5 scaling because we open two files per bucket? Seems very tight if so. // Give up if we hit our max limit. if ((minMemory > maxMemory) || (numOverlaps / olapsPerBucketMax + 1) > 0.50 * maxFiles) { fprintf(stderr, "ERROR: Cannot sort %.2f million overlaps using %.2f GB memory; too few file handles available.\n", numOverlaps / 1000000.0, maxMemory / 1024.0 / 1024.0 / 1024.0); fprintf(stderr, "ERROR: olapsPerBucket "F_U64"\n", olapsPerBucketMax); fprintf(stderr, "ERROR: buckets "F_U64"\n", numOverlaps / olapsPerBucketMax + 1); fprintf(stderr, "ERROR: Increase memory size (in canu, ovsMemory; in ovStoreBuild, -M)\n"); exit(1); } fprintf(stderr, "Will sort using "F_U64" files; "F_U64" (%.2f million) overlaps per bucket; %.2f GB memory per bucket\n", numOverlaps / olapsPerBucketMax + 1, olapsPerBucketMax, olapsPerBucketMax / 1000000.0, olapsPerBucketMax * GBperOlap + MEMORY_OVERHEAD / 1024.0 / 1024.0 / 1024.0); } // Given the limit on each bucket, count the number of buckets needed, then reset the limit on // each bucket to have the same number of overlaps for every bucket. { uint64 olaps = 0; uint32 bucket = 1; for (uint32 ii=0; ii= olapsPerBucketMax) { olaps = 0; bucket++; } } olapsPerBucketMax = (uint64)ceil((double)numOverlaps / (double)bucket); } // And, finally, assign IIDs to buckets. { uint64 olaps = 0; uint32 bucket = 1; for (uint32 ii=0; ii= olapsPerBucketMax) { fprintf(stderr, " bucket %3d has "F_U64" olaps.\n", bucket, olaps); olaps = 0; bucket++; } } fprintf(stderr, " bucket %3d has "F_U64" olaps.\n", bucket, olaps); } fprintf(stderr, "Will sort %.3f million overlaps per bucket, using %u buckets %.2f GB per bucket.\n", olapsPerBucketMax / 1000000.0, iidToBucket[maxIID-1], olapsPerBucketMax * GBperOlap + MEMORY_OVERHEAD / 1024.0 / 1024.0 / 1024.0); delete [] overlapsPerRead; return(iidToBucket); } static void writeToDumpFile(ovOverlap *overlap, ovFile **dumpFile, uint64 *dumpLength, uint32 *iidToBucket, char *ovlName) { uint32 df = iidToBucket[overlap->a_iid]; // If the dump file isn't opened, open it. if (dumpFile[df] == NULL) { char name[FILENAME_MAX]; sprintf(name, "%s/tmp.sort.%03d", ovlName, df); fprintf(stderr, "CREATE bucket '%s'\n", name); dumpFile[df] = new ovFile(name, ovFileFullWriteNoCounts); dumpLength[df] = 0; } // And write the overlap. dumpFile[df]->writeOverlap(overlap); dumpLength[df]++; } int main(int argc, char **argv) { char *ovlName = NULL; char *gkpName = NULL; uint32 fileLimit = 0; uint64 minMemory = (uint64)1 * 1024 * 1024 * 1024; uint64 maxMemory = (uint64)4 * 1024 * 1024 * 1024; double maxError = 1.0; uint32 minOverlap = 0; vector fileList; uint32 nThreads = 4; bool eValues = false; char *configOut = NULL; argc = AS_configure(argc, argv); int err=0; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-O") == 0) { ovlName = argv[++arg]; } else if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-F") == 0) { fileLimit = atoi(argv[++arg]); minMemory = 0; maxMemory = 0; } else if (strcmp(argv[arg], "-M") == 0) { double lo=0.0, hi=0.0; AS_UTL_decodeRange(argv[++arg], lo, hi); minMemory = (uint64)ceil(lo * 1024.0 * 1024.0 * 1024.0); maxMemory = (uint64)ceil(hi * 1024.0 * 1024.0 * 1024.0); fileLimit = 0; } else if (strcmp(argv[arg], "-e") == 0) { maxError = atof(argv[++arg]); } else if (strcmp(argv[arg], "-e") == 0) { minOverlap = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-L") == 0) { AS_UTL_loadFileList(argv[++arg], fileList); } else if (strcmp(argv[arg], "-evalues") == 0) { eValues = true; } else if (strcmp(argv[arg], "-config") == 0) { configOut = argv[++arg]; } else if (((argv[arg][0] == '-') && (argv[arg][1] == 0)) || (AS_UTL_fileExists(argv[arg]))) { // Assume it's an input file fileList.push_back(argv[arg]); } else { fprintf(stderr, "%s: unknown option '%s'.\n", argv[0], argv[arg]); err++; } arg++; } if (ovlName == NULL) err++; if (gkpName == NULL) err++; if (fileList.size() == 0) err++; if (fileLimit > sysconf(_SC_OPEN_MAX) - 16) err++; if (maxMemory < MEMORY_OVERHEAD) err++; if (err) { fprintf(stderr, "usage: %s -O asm.ovlStore -G asm.gkpStore [opts] [-L fileList | *.ovb.gz]\n", argv[0]); fprintf(stderr, " -O asm.ovlStore path to store to create\n"); fprintf(stderr, " -G asm.gkpStore path to gkpStore for this assembly\n"); fprintf(stderr, "\n"); fprintf(stderr, " -L fileList read input filenames from 'flieList'\n"); fprintf(stderr, "\n"); fprintf(stderr, " -F f use up to 'f' files for store creation\n"); fprintf(stderr, " -M g use up to 'g' gigabytes memory for sorting overlaps\n"); fprintf(stderr, " default 4; g-0.25 gb is available for sorting overlaps\n"); fprintf(stderr, "\n"); fprintf(stderr, " -e e filter overlaps above e fraction error\n"); fprintf(stderr, " -l l filter overlaps below l bases overlap length (needs gkpStore to get read lengths!)\n"); fprintf(stderr, "\n"); fprintf(stderr, "Non-building options:\n"); fprintf(stderr, " -evalues input files are evalue updates from overlap error adjustment\n"); fprintf(stderr, " -config out.dat don't build a store, just dump a binary partitioning file for ovStoreBucketizer\n"); fprintf(stderr, "\n"); if (ovlName == NULL) fprintf(stderr, "ERROR: No overlap store (-o) supplied.\n"); if (gkpName == NULL) fprintf(stderr, "ERROR: No gatekeeper store (-g) supplied.\n"); if (fileList.size() == 0) fprintf(stderr, "ERROR: No input overlap files (-L or last on the command line) supplied.\n"); if (fileLimit > sysconf(_SC_OPEN_MAX) - 16) fprintf(stderr, "ERROR: Too many jobs (-F); only "F_SIZE_T" supported on this architecture.\n", sysconf(_SC_OPEN_MAX) - 16); if (maxMemory < MEMORY_OVERHEAD) fprintf(stderr, "ERROR: Memory (-M) must be at least %.3f GB to account for overhead.\n", MEMORY_OVERHEAD / 1024.0 / 1024.0 / 1024.0); exit(1); } if (eValues) { ovStore *ovs = new ovStore(ovlName, NULL); for (uint32 i=0; iaddEvalues(bgnID, endID, evalues, len); delete [] evalues; } delete ovs; exit(0); } // Open reads, figure out a partitioning scheme. gkStore *gkp = gkStore::gkStore_open(gkpName); uint64 maxIID = gkp->gkStore_getNumReads() + 1; uint32 *iidToBucket = computeIIDperBucket(fileLimit, minMemory, maxMemory, maxIID, fileList); uint32 maxFiles = sysconf(_SC_OPEN_MAX); if (iidToBucket[maxIID-1] > maxFiles - 8) { fprintf(stderr, "ERROR:\n"); fprintf(stderr, "ERROR: Operating system limit of "F_U32" open files. The current -F/-M settings\n", maxFiles); fprintf(stderr, "ERROR: will need to create "F_U32" files to construct the store.\n", iidToBucket[maxIID-1]); fprintf(stderr, "ERROR:\n"); exit(1); } // Dump the configuration if told to. if (configOut) { errno = 0; FILE *C = fopen(configOut, "w"); if (errno) fprintf(stderr, "Failed to open config output file '%s': %s\n", configOut, strerror(errno)), exit(1); AS_UTL_safeWrite(C, &maxIID, "maxIID", sizeof(uint32), 1); AS_UTL_safeWrite(C, iidToBucket, "iidToBucket", sizeof(uint32), maxIID); fclose(C); delete [] iidToBucket; gkp->gkStore_close(); fprintf(stderr, "saved configuration to '%s'.\n", configOut); exit(0); } // Read the gkStore to determine which fragments we care about. ovStoreFilter *filter = new ovStoreFilter(gkp, maxError); // And load reads into the store! We used to create the store before filtering, so it could fail // quicker, but the filter should be much faster with the mmap()'d gkpStore in canu. ovStore *storeFile = new ovStore(ovlName, gkp, ovStoreWrite); uint32 dumpFileMax = iidToBucket[maxIID-1] + 1; ovFile **dumpFile = new ovFile * [dumpFileMax]; uint64 *dumpLength = new uint64 [dumpFileMax]; memset(dumpFile, 0, sizeof(ovFile *) * dumpFileMax); memset(dumpLength, 0, sizeof(uint64) * dumpFileMax); for (uint32 i=0; ireadOverlap(&foverlap)) { filter->filterOverlap(foverlap, roverlap); // The filter copies f into r // Check that overlap IDs are valid. #warning not checking overlap IDs for validity // If all are skipped, don't bother writing the overlap. if ((foverlap.dat.ovl.forUTG == true) || (foverlap.dat.ovl.forOBT == true) || (foverlap.dat.ovl.forDUP == true)) writeToDumpFile(&foverlap, dumpFile, dumpLength, iidToBucket, ovlName); if ((roverlap.dat.ovl.forUTG == true) || (roverlap.dat.ovl.forOBT == true) || (roverlap.dat.ovl.forDUP == true)) writeToDumpFile(&roverlap, dumpFile, dumpLength, iidToBucket, ovlName); } delete inputFile; // AFTER EVERY FILE filter->reportFate(); filter->resetCounters(); } delete filter; for (uint32 i=0; ireadOverlap(overlapsort + numOvl)) { // Quick sanity check on IIDs. if ((overlapsort[numOvl].a_iid == 0) || (overlapsort[numOvl].b_iid == 0) || (overlapsort[numOvl].a_iid >= maxIID) || (overlapsort[numOvl].b_iid >= maxIID)) { char ovlstr[256]; fprintf(stderr, "Overlap has IDs out of range (maxIID "F_U64"), possibly corrupt input data.\n", maxIID); fprintf(stderr, " Aid "F_U32" Bid "F_U32"\n", overlapsort[numOvl].a_iid, overlapsort[numOvl].b_iid); exit(1); } numOvl++; } delete bof; assert(numOvl == dumpLength[i]); assert(numOvl <= dumpLengthMax); // There's no real advantage to saving this file until after we write it out. If we crash // anywhere during the build, we are forced to restart from scratch. I'll argue that removing // it early helps us to not crash from running out of disk space. unlink(name); fprintf(stderr, "sorting %s (%ld)\n", name, time(NULL) - beginTime); #ifdef _GLIBCXX_PARALLEL // If we have the parallel STL, don't use it! Sort is not inplace! __gnu_sequential::sort(overlapsort, overlapsort + dumpLength[i]); #else sort(overlapsort, overlapsort + dumpLength[i]); #endif fprintf(stderr, "writing %s (%ld)\n", name, time(NULL) - beginTime); for (uint64 x=0; xwriteOverlap(overlapsort + x); } delete storeFile; delete [] overlapsort; gkp->gkStore_close(); // And we have a store. exit(0); } canu-1.3/src/stores/ovStoreBuild.mk000066400000000000000000000007311272335422300173670ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := ovStoreBuild SOURCES := ovStoreBuild.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/ovStoreDump.C000066400000000000000000000575661272335422300170320ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVS/overlapStore.C * src/AS_OVS/overlapStore.c * * Modifications by: * * Brian P. Walenz from 2007-MAR-08 to 2013-AUG-01 * are Copyright 2007-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-DEC-16 to 2009-AUG-14 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Gregory Sims from 2012-FEB-01 to 2012-MAR-14 * are Copyright 2012 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-AUG-22 to 2015-JUN-25 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-21 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-MAR-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "AS_UTL_decodeRange.H" #include "splitToWords.H" #include "gkStore.H" #include "ovStore.H" enum dumpOp { OP_NONE = 1, OP_DUMP = 2, OP_DUMP_PICTURE = 3 }; enum dumpFlags { NO_5p = 1, NO_3p = 2, NO_CONTAINED = 4, NO_CONTAINS = 8, NO_CONTAINED_READS = 16, NO_SUSPICIOUS_READS = 32, NO_SINGLETON_READS = 64 }; struct readStatus { uint64 best5id : 29; uint64 best53p : 1; // Unwieldy - best edge from my 5' is to the 3' of 'best5id'. uint64 best3id : 29; uint64 best33p : 1; uint64 unused : 1; uint64 isSingleton : 1; uint64 isContained : 1; uint64 isSuspicious : 1; }; class bogartStatus { public: bogartStatus(const char *prefix, uint32 nReads); ~bogartStatus() { delete [] _status; }; uint32 getBest5id(uint32 id) { return((_status) ? (_status[id].best5id) : 0); }; bool getBest53p(uint32 id) { return((_status) ? (_status[id].best53p) : 0); }; uint32 getBest3id(uint32 id) { return((_status) ? (_status[id].best3id) : false); }; bool getBest33p(uint32 id) { return((_status) ? (_status[id].best33p) : false); }; bool getSingleton(uint32 id) { return((_status) ? (_status[id].isSingleton) : false); }; bool getContained(uint32 id) { return((_status) ? (_status[id].isContained) : false); }; bool getSuspicious(uint32 id) { return((_status) ? (_status[id].isSuspicious) : false); }; private: readStatus *_status; }; bogartStatus::bogartStatus(const char *prefix, uint32 nReads) { char N[FILENAME_MAX]; splitToWords W; _status = NULL; if (prefix == NULL) return; errno = 0; sprintf(N, "%s.edges", prefix); FILE *E = fopen(N, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", N, strerror(errno)), exit(1); sprintf(N, "%s.edges.suspicious", prefix); FILE *S = fopen(N, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", N, strerror(errno)), exit(1); sprintf(N, "%s.singletons", prefix); FILE *G = fopen(N, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", N, strerror(errno)), exit(1); _status = new readStatus [nReads+1]; memset(_status, 0, sizeof(readStatus) * (nReads+1)); fgets(N, FILENAME_MAX, E); while (!feof(E)) { W.split(N); uint32 id = W(0); _status[id].best5id = W(2); _status[id].best53p = (W[3][0] == '3'); _status[id].best3id = W(4); _status[id].best33p = (W[5][0] == '3'); _status[id].isSingleton = false; _status[id].isContained = ((W.numWords() > 10) && (W[10][0] == 'c')); _status[id].isSuspicious = false; fgets(N, FILENAME_MAX, E); } fclose(E); fgets(N, FILENAME_MAX, S); while (!feof(S)) { W.split(N); uint32 id = W(0); _status[id].best5id = W(2); _status[id].best53p = (W[3][0] == '3'); _status[id].best3id = W(4); _status[id].best33p = (W[5][0] == '3'); _status[id].isSingleton = false; _status[id].isContained = ((W.numWords() > 10) && (W[10][0] == 'c')); _status[id].isSuspicious = true; fgets(N, FILENAME_MAX, S); } fclose(S); fgets(N, FILENAME_MAX, G); while (!feof(G)) { W.split(N); uint32 id = W(0); _status[id].best5id = 0; _status[id].best53p = 0; _status[id].best3id = 0; _status[id].best33p = 0; _status[id].isSingleton = true; _status[id].isContained = false; _status[id].isSuspicious = false; fgets(N, FILENAME_MAX, G); } fclose(G); } // // Also accept a single ovStoreFile (output from overlapper) and dump. // // // Then need some way of loading ascii overlaps into a store, or converting ascii overlaps to // binary and use the normal store build. The normal store build also needs to take sorted // overlaps and just rewrite as a store. // void dumpStore(ovStore *ovlStore, gkStore *gkpStore, bool asBinary, bool asCounts, double dumpERate, uint32 dumpType, uint32 dumpLength, uint32 bgnID, uint32 endID, uint32 qryID, ovOverlapDisplayType type, bool beVerbose, bool oneSided, char *bestPrefix) { ovOverlap overlap(gkpStore); uint64 evalue = AS_OVS_encodeEvalue(dumpERate); char ovlString[1024]; uint32 ovlTooHighError = 0; uint32 ovlNot5p = 0; uint32 ovlNot3p = 0; uint32 ovlNotContainer = 0; uint32 ovlNotContainee = 0; uint32 ovlNotUnique = 0; uint32 ovlDumped = 0; uint32 obtTooHighError = 0; uint32 obtDumped = 0; uint32 merDumped = 0; uint32 *counts = (asCounts) ? new uint32 [endID - bgnID + 1] : NULL; if (asCounts) for (uint32 ii=bgnID; ii<=endID; ii++) counts[ii - bgnID] = 0; ovlStore->setRange(bgnID, endID); // Length filtering is expensive to compute, need to load both reads to get their length. // //if ((dumpLength > 0) && (dumpLength < overlapLength(overlap))) // continue; while (ovlStore->readOverlap(&overlap) == TRUE) { if ((qryID != 0) && (qryID != overlap.b_iid)) continue; if (overlap.evalue() > evalue) { ovlTooHighError++; continue; } int32 ahang = overlap.a_hang(); int32 bhang = overlap.b_hang(); if ((dumpType & NO_5p) && (ahang < 0) && (bhang < 0)) { ovlNot5p++; continue; } if ((dumpType & NO_3p) && (ahang > 0) && (bhang > 0)) { ovlNot3p++; continue; } if ((dumpType & NO_CONTAINS) && (ahang >= 0) && (bhang <= 0)) { ovlNotContainer++; continue; } if ((dumpType & NO_CONTAINED) && (ahang <= 0) && (bhang >= 0)) { ovlNotContainee++; continue; } if (oneSided == true && overlap.a_iid >= overlap.b_iid) { ovlNotUnique++; continue; } ovlDumped++; // The toString() method is quite slow, all from sprintf(). // Without both the puts() and AtoString(), a dump ran in 3 seconds. // With both, 138 seconds. // Without the puts(), 127 seconds. if (asCounts) counts[overlap.a_iid - bgnID]++; else if (asBinary) AS_UTL_safeWrite(stdout, &overlap, "dumpStore", sizeof(ovOverlap), 1); else fputs(overlap.toString(ovlString, type, true), stdout); } if (asCounts) for (uint32 ii=bgnID; ii<=endID; ii++) fprintf(stdout, "%u\t%u\n", ii, counts[ii - bgnID]); delete [] counts; if (beVerbose) { fprintf(stderr, "ovlTooHighError %u\n", ovlTooHighError); fprintf(stderr, "ovlNot5p %u\n", ovlNot5p); fprintf(stderr, "ovlNot3p %u\n", ovlNot3p); fprintf(stderr, "ovlNotContainer %u\n", ovlNotContainer); fprintf(stderr, "ovlNotContainee %u\n", ovlNotContainee); fprintf(stderr, "ovlDumped %u\n", ovlDumped); fprintf(stderr, "obtTooHighError %u\n", obtTooHighError); fprintf(stderr, "obtDumped %u\n", obtDumped); fprintf(stderr, "merDumped %u\n", merDumped); } } int sortOBT(const void *a, const void *b) { ovOverlap const *A = (ovOverlap const *)a; ovOverlap const *B = (ovOverlap const *)b; if (A->a_bgn() < B->a_bgn()) return(-1); if (A->a_bgn() > B->a_bgn()) return(1); // For overlaps off the 5' end, put the thinnest ones first. Sadly, we don't know // gkpStore here, and can't actually get the end coordinate. // //if (A->b_bgn() > B->b_bgn()) return(-1); //if (A->b_bgn() < B->b_bgn()) return(1); if (A->a_end() < B->a_end()) return(-1); if (A->a_end() > B->a_end()) return(1); return(0); } void dumpPicture(ovOverlap *overlaps, uint64 novl, gkStore *gkpStore, uint32 qryID, bogartStatus *bogart) { char ovl[256] = {0}; uint32 MHS = 7; // Max Hang Size, amount of padding for "+### " uint32 Aid = qryID; gkRead *A = gkpStore->gkStore_getRead(Aid); uint32 frgLenA = A->gkRead_sequenceLength(); for (int32 i=0; i<256; i++) ovl[i] = ' '; for (int32 i=0; i<100; i++) ovl[i + MHS] = '-'; ovl[ 99 + MHS] = '>'; ovl[100 + MHS] = 0; fprintf(stdout, "%8d A: %5d %5d %s %s%s\n", Aid, 0, frgLenA, ovl, bogart->getContained(Aid) ? "contained" : "", bogart->getSuspicious(Aid) ? "suspicious" : ""); qsort(overlaps, novl, sizeof(ovOverlap), sortOBT); // Build ascii representations for each overlapping read. for (uint32 o=0; ogkStore_getRead(Bid); uint32 frgLenB = B->gkRead_sequenceLength(); // Find bgn/end points on each read. If the overlap is reverse complement, // the B coords are flipped so that bgn > end. uint32 ovlBgnA = overlaps[o].a_bgn(); uint32 ovlEndA = overlaps[o].a_end(); uint32 ovlBgnB = overlaps[o].b_bgn(); uint32 ovlEndB = overlaps[o].b_end(); assert(ovlBgnA < ovlEndA); // The A coordiantes are always forward if (overlaps[o].flipped() == false) assert(ovlBgnB < ovlEndB); // Forward overlaps are forward else assert(ovlEndB < ovlBgnB); // Flipped overlaps are reversed // For the A read, find the points in our string representation where the overlap ends. uint32 ovlStrBgn = ovlBgnA * 100 / frgLenA + MHS; uint32 ovlStrEnd = ovlEndA * 100 / frgLenA + MHS; // Fill the string representation with spaces, then fill the string with dashes where the read // is, add an arrow, and terminate the string. for (int32 i=0; i<256; i++) ovl[i] = ' '; // Decide how to draw this overlap. // For best edges, use '='. // For contained, use '-', alternating with spaces. // For suspicious, use '*', alternating with dashes. // For edges, use '-', solid. bool isBest = (((bogart->getBest5id(Aid) == Bid) && (overlaps[o].overlapAEndIs5prime() == true)) || ((bogart->getBest3id(Aid) == Bid) && (overlaps[o].overlapAEndIs3prime() == true))); bool isCont = (bogart->getContained(Bid)); bool isSusp = (bogart->getSuspicious(Bid)); // This bit of confusion makes sure that the alternating overlap lines (especially '- - - -') // end with a dash. bool oddEven = (overlaps[o].flipped() == false) ? (false) : (((ovlStrEnd - ovlStrBgn) % 2) == false); if (isCont == true) { for (uint32 i=ovlStrBgn; i 0) { char str[256]; int32 len; sprintf(str, "+%d", ovlBgnHang); len = strlen(str); for (int32 i=0; i 0) { sprintf(ovl + ovlStrEnd, " +%d", ovlEndHang); } // Set flags for best edge and singleton/contained/suspicious. Left in for when I get annoyed with the different lines. char olapClass[4] = { 0, ' ', 0, 0 }; #if 0 if ((bogart->getBest5id(Aid) == Bid) && (overlaps[o].overlapAEndIs5prime() == true)) { olapClass[0] = ' '; olapClass[2] = 'B'; } if ((bogart->getBest3id(Aid) == Bid) && (overlaps[o].overlapAEndIs3prime() == true)) { olapClass[0] = ' '; olapClass[2] = 'B'; } if (olapClass[2] == 'B') for (uint32 ii=0; ovl[ii]; ii++) if (ovl[ii] == '-') ovl[ii] = '='; if (bogart->getSingleton(Bid)) { olapClass[0] = ' '; olapClass[1] = 'S'; } if (bogart->getContained(Bid)) { olapClass[0] = ' '; olapClass[1] = 'C'; } if (bogart->getSuspicious(Bid)) { olapClass[0] = ' '; olapClass[1] = '!'; } #endif // Report! fprintf(stdout, "%8d A: %5d %5d (%5d) B: %5d %5d (%5d) %5.2f%% %s%s\n", Bid, ovlBgnA, ovlEndA, frgLenA, ovlBgnB, ovlEndB, frgLenB, overlaps[o].erate() * 100.0, ovl, olapClass); } } void dumpPicture(ovStore *ovlStore, gkStore *gkpStore, double dumpERate, uint32 dumpLength, uint32 dumpType, uint32 qryID, char *bestPrefix) { //fprintf(stderr, "DUMPING PICTURE for ID "F_U32" in store %s (gkp %s)\n", // qryID, ovlName, gkpName); uint32 Aid = qryID; gkRead *A = gkpStore->gkStore_getRead(Aid); uint32 frgLenA = A->gkRead_sequenceLength(); ovlStore->setRange(Aid, Aid); uint64 novl = 0; ovOverlap overlap(gkpStore); ovOverlap *overlaps = ovOverlap::allocateOverlaps(gkpStore, ovlStore->numOverlapsInRange()); uint64 evalue = AS_OVS_encodeEvalue(dumpERate); // Load bogart status, if supplied. bogartStatus *bogart = new bogartStatus(bestPrefix, gkpStore->gkStore_getNumReads()); // Load all the overlaps so we can sort by the A begin position. while (ovlStore->readOverlap(&overlap) == TRUE) { // Filter out garbage overlaps. if (overlap.evalue() > evalue) continue; // Filter out 5' overlaps. if ((dumpType & NO_5p) && (overlap.a_hang() < 0) && (overlap.b_hang() < 0)) continue; // Filter out 3' overlaps. if ((dumpType & NO_3p) && (overlap.a_hang() > 0) && (overlap.b_hang() > 0)) continue; // Filter out contained overlaps (B-read is contained) if ((dumpType & NO_CONTAINS) && (overlap.a_hang() >= 0) && (overlap.b_hang() <= 0)) continue; // Filter out container overlaps (A-read is contained) if ((dumpType & NO_CONTAINED) && (overlap.a_hang() <= 0) && (overlap.b_hang() >= 0)) continue; // Filter out short overlaps. if ((overlap.b_end() - overlap.b_bgn() < dumpLength) || (overlap.a_end() - overlap.a_bgn() < dumpLength)) continue; // If bogart data is supplied, filter out contained or suspicious overlaps. if ((dumpType & NO_CONTAINED_READS) && (bogart->getContained(overlap.b_iid))) continue; if ((dumpType & NO_SUSPICIOUS_READS) && (bogart->getSuspicious(overlap.b_iid))) continue; if ((dumpType & NO_SINGLETON_READS) && (bogart->getSingleton(overlap.b_iid))) continue; overlaps[novl++] = overlap; } if (novl == 0) fprintf(stderr, "no overlaps to show.\n"); else dumpPicture(overlaps, novl, gkpStore, Aid, bogart); delete [] overlaps; } int main(int argc, char **argv) { uint32 operation = OP_NONE; char *gkpName = NULL; char *ovlName = NULL; bool asBinary = false; bool asCounts = false; double dumpERate = 1.0; uint32 dumpLength = 0; uint32 dumpType = 0; char *erateFile = NULL; uint32 bgnID = 0; uint32 endID = UINT32_MAX; uint32 qryID = 0; bool beVerbose = false; bool oneSided = false; char *bestPrefix = NULL; ovOverlapDisplayType type = ovOverlapAsCoords; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) gkpName = argv[++arg]; else if (strcmp(argv[arg], "-O") == 0) ovlName = argv[++arg]; // Standard bulk dump of overlaps else if (strcmp(argv[arg], "-d") == 0) { operation = OP_DUMP; if ((arg+1 < argc) && (argv[arg+1][0] != '-')) AS_UTL_decodeRange(argv[++arg], bgnID, endID); } // Dump as a picture else if (strcmp(argv[arg], "-p") == 0) { operation = OP_DUMP_PICTURE; bgnID = atoi(argv[++arg]); endID = bgnID; qryID = bgnID; } // Query if the overlap for the next two integers exists else if (strcmp(argv[arg], "-q") == 0) { operation = OP_DUMP; bgnID = atoi(argv[++arg]); endID = bgnID; qryID = atoi(argv[++arg]); } // Format of the dump else if (strcmp(argv[arg], "-coords") == 0) type = ovOverlapAsCoords; else if (strcmp(argv[arg], "-hangs") == 0) type = ovOverlapAsHangs; else if (strcmp(argv[arg], "-raw") == 0) type = ovOverlapAsRaw; else if (strcmp(argv[arg], "-paf") == 0) type = ovOverlapAsPaf; else if (strcmp(argv[arg], "-binary") == 0) asBinary = true; else if (strcmp(argv[arg], "-counts") == 0) asCounts = true; // standard bulk dump options else if (strcmp(argv[arg], "-E") == 0) dumpERate = atof(argv[++arg]); else if (strcmp(argv[arg], "-L") == 0) dumpLength = atoi(argv[++arg]); else if (strcmp(argv[arg], "-d5") == 0) dumpType |= NO_5p; else if (strcmp(argv[arg], "-d3") == 0) dumpType |= NO_3p; else if (strcmp(argv[arg], "-dC") == 0) dumpType |= NO_CONTAINS; else if (strcmp(argv[arg], "-dc") == 0) dumpType |= NO_CONTAINED; else if (strcmp(argv[arg], "-v") == 0) beVerbose = true; else if (strcmp(argv[arg], "-unique") == 0) oneSided = true; else if (strcmp(argv[arg], "-best") == 0) bestPrefix = argv[++arg]; else if (strcmp(argv[arg], "-noc") == 0) dumpType |= NO_CONTAINED_READS; else if (strcmp(argv[arg], "-nos") == 0) dumpType |= NO_SUSPICIOUS_READS; else if (strcmp(argv[arg], "-nosi") == 0) dumpType |= NO_SINGLETON_READS; else { fprintf(stderr, "%s: unknown option '%s'.\n", argv[0], argv[arg]); err++; } arg++; } if (operation == OP_NONE) err++; if (gkpName == NULL) err++; if (ovlName == NULL) err++; if (err) { fprintf(stderr, "usage: %s -G gkpStore -O ovlStore ...\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, "There are three modes of operation:\n"); fprintf(stderr, " -d [a[-b]] dump overlaps for reads a to b, inclusive\n"); fprintf(stderr, " -q a b report the a,b overlap, if it exists.\n"); fprintf(stderr, " -p a dump a picture of overlaps to fragment 'a'.\n"); fprintf(stderr, "\n"); fprintf(stderr, " FORMAT (for -d)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -coords dump overlap showing coordinates in the reads (default)\n"); fprintf(stderr, " -hangs dump overlap showing dovetail hangs unaligned\n"); fprintf(stderr, " -raw dump overlap showing its raw native format (four hangs)\n"); fprintf(stderr, " -paf dump overlaps in miniasm/minimap format\n"); fprintf(stderr, " -binary dump overlap as raw binary data\n"); fprintf(stderr, " -counts dump the number of overlaps per read\n"); fprintf(stderr, "\n"); fprintf(stderr, " MODIFIERS (for -d and -p)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -E erate Dump only overlaps <= erate fraction error.\n"); fprintf(stderr, " -L length Dump only overlaps that are larger than L bases (only for -p picture mode).\n"); fprintf(stderr, " -d5 Dump only overlaps off the 5' end of the A frag.\n"); fprintf(stderr, " -d3 Dump only overlaps off the 3' end of the A frag.\n"); fprintf(stderr, " -dC Dump only overlaps that are contained in the A frag (B contained in A).\n"); fprintf(stderr, " -dc Dump only overlaps that are containing the A frag (A contained in B).\n"); fprintf(stderr, " -v Report statistics (to stderr) on some dumps (-d).\n"); fprintf(stderr, " -unique Report only overlaps where A id is < B id, do not report both A to B and B to A overlap\n"); fprintf(stderr, "\n"); fprintf(stderr, " -best prefix Annotate picture with status from bogart outputs prefix.edges, prefix.singletons, prefix.edges.suspicious\n"); fprintf(stderr, " -noc With -best data, don't show overlaps to contained reads.\n"); fprintf(stderr, " -nos With -best data, don't show overlaps to suspicious reads.\n"); fprintf(stderr, "\n"); if (operation == OP_NONE) fprintf(stderr, "ERROR: no operation (-d, -q or -p) supplied.\n"); if (gkpName == NULL) fprintf(stderr, "ERROR: no input gkpStore (-G) supplied.\n"); if (ovlName == NULL) fprintf(stderr, "ERROR: no input ovlStore (-O) supplied.\n"); exit(1); } gkStore *gkpStore = gkStore::gkStore_open(gkpName); ovStore *ovlStore = new ovStore(ovlName, gkpStore); if (endID > gkpStore->gkStore_getNumReads()) endID = gkpStore->gkStore_getNumReads(); if (endID < bgnID) fprintf(stderr, "ERROR: invalid bgn/end range bgn=%u end=%u; only %u reads in the store\n", bgnID, endID, gkpStore->gkStore_getNumReads()), exit(1); switch (operation) { case OP_DUMP: dumpStore(ovlStore, gkpStore, asBinary, asCounts, dumpERate, dumpLength, dumpType, bgnID, endID, qryID, type, beVerbose, oneSided, bestPrefix); break; case OP_DUMP_PICTURE: for (qryID=bgnID; qryID <= endID; qryID++) dumpPicture(ovlStore, gkpStore, dumpERate, dumpLength, dumpType, qryID, bestPrefix); break; default: break; } delete ovlStore; gkpStore->gkStore_close(); exit(0); } canu-1.3/src/stores/ovStoreDump.mk000066400000000000000000000007271272335422300172420ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := ovStoreDump SOURCES := ovStoreDump.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/ovStoreFile.C000066400000000000000000000327701272335422300167720ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVS/AS_OVS_overlapFile.C * src/AS_OVS/AS_OVS_overlapFile.c * * Modifications by: * * Brian P. Walenz from 2007-FEB-28 to 2013-SEP-22 * are Copyright 2007-2009,2011-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2011-MAR-31 * are Copyright 2011 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Gregory Sims on 2012-FEB-01 * are Copyright 2012 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-09 to 2015-JUN-16 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "ovStore.H" ovFile::ovFile(const char *name, ovFileType type, uint32 bufferSize) { // We write two sizes of overlaps. The 'normal' format doesn't contain the a_iid, while the // 'full' format does. The buffer size must hold an integer number of overlaps, otherwise the // reader will read partial overlaps and fail. Choose a buffer size that can handle both. uint32 lcm = ((sizeof(uint32) * 1 + sizeof(ovOverlapDAT)) * (sizeof(uint32) * 2 + sizeof(ovOverlapDAT))); if (bufferSize < 16 * 1024) bufferSize = 16 * 1024; _bufferLen = 0; _bufferPos = (bufferSize / (lcm * sizeof(uint32))) * lcm; // Forces reload on next read _bufferMax = (bufferSize / (lcm * sizeof(uint32))) * lcm; _buffer = new uint32 [_bufferMax]; assert(_bufferMax % ((sizeof(uint32) * 1) + (sizeof(ovOverlapDAT))) == 0); assert(_bufferMax % ((sizeof(uint32) * 2) + (sizeof(ovOverlapDAT))) == 0); // When writing full overlaps, we also write the number of overlaps per read. This is used to // build the store. Overlaps in the store, normal format, don't need this extra data, as the // store itself knows how many overlaps per read. _olapsPerReadAlloc = 0; _olapsPerReadLast = 0; _olapsPerRead = NULL; if (type == ovFileFullWrite) { _olapsPerReadAlloc = 128 * 1024; _olapsPerReadLast = 0; _olapsPerRead = new uint32 [_olapsPerReadAlloc]; memset(_olapsPerRead, 0, sizeof(uint32) * _olapsPerReadAlloc); } // Create the input/output buffers and files. _isOutput = false; _isSeekable = false; _isNormal = (type == ovFileNormal) || (type == ovFileNormalWrite); _reader = NULL; _writer = NULL; // Open a file for reading? if ((type == ovFileNormal) || (type == ovFileFull)) { _reader = new compressedFileReader(name); _file = _reader->file(); _isSeekable = (_reader->isCompressed() == false); } // Open a file for writing? else { _writer = new compressedFileWriter(name); _file = _writer->file(); _isOutput = true; } // Make a copy of the output name, and clean it up. This is used as the base for // the counts output. We just strip off all the dotted extensions in the filename. strcpy(_prefix, name); char *slash = strrchr(_prefix, '/'); char *dot = strchr((slash == NULL) ? _prefix : slash, '.'); if (dot) *dot = 0; } ovFile::~ovFile() { flushOverlaps(); delete _reader; delete _writer; delete [] _buffer; if (_olapsPerRead) { char name[FILENAME_MAX]; sprintf(name, "%s.counts", _prefix); errno = 0; _file = fopen(name, "w"); if (errno) fprintf(stderr, "failed to open counts file '%s' for writing: %s\n", name, strerror(errno)), exit(1); _olapsPerReadLast++; AS_UTL_safeWrite(_file, &_olapsPerReadLast, "ovFile::olapsPerReadLast", sizeof(uint32), 1); AS_UTL_safeWrite(_file, _olapsPerRead, "ovFile::olapsPerRead", sizeof(uint32), _olapsPerReadLast); fclose(_file); delete [] _olapsPerRead; //fprintf(stderr, "Wrote counts file '%s' for reads up to iid "F_U32"\n", name, _olapsPerReadLast); } } void ovFile::flushOverlaps(void) { if (_isOutput == false) return; if (_bufferLen == 0) return; AS_UTL_safeWrite(_file, _buffer, "ovFile::flushOverlaps", sizeof(uint32), _bufferLen); _bufferLen = 0; } void ovFile::writeOverlap(ovOverlap *overlap) { assert(_isOutput == true); if (_bufferLen >= _bufferMax) { AS_UTL_safeWrite(_file, _buffer, "ovFile::writeOverlap", sizeof(uint32), _bufferLen); _bufferLen = 0; } if (_olapsPerRead) { uint32 newmax = _olapsPerReadAlloc; uint32 newlast = _olapsPerReadLast; newlast = max(newlast, overlap->a_iid); newlast = max(newlast, overlap->b_iid); while (newmax <= newlast) newmax += newmax / 4; resizeArray(_olapsPerRead, _olapsPerReadLast+1, _olapsPerReadAlloc, newmax, resizeArray_copyData | resizeArray_clearNew); _olapsPerRead[overlap->a_iid]++; _olapsPerRead[overlap->b_iid]++; _olapsPerReadLast = newlast; } if (_isNormal == false) _buffer[_bufferLen++] = overlap->a_iid; _buffer[_bufferLen++] = overlap->b_iid; #if (ovOverlapNWORDS == 5) _buffer[_bufferLen++] = overlap->dat.dat[0]; _buffer[_bufferLen++] = overlap->dat.dat[1]; _buffer[_bufferLen++] = overlap->dat.dat[2]; _buffer[_bufferLen++] = overlap->dat.dat[3]; _buffer[_bufferLen++] = overlap->dat.dat[4]; #elif (ovOverlapNWORDS == 3) _buffer[_bufferLen++] = (overlap->dat.dat[0] >> 32) & 0xffffffff; _buffer[_bufferLen++] = (overlap->dat.dat[0] >> 0) & 0xffffffff; _buffer[_bufferLen++] = (overlap->dat.dat[1] >> 32) & 0xffffffff; _buffer[_bufferLen++] = (overlap->dat.dat[1] >> 0) & 0xffffffff; _buffer[_bufferLen++] = (overlap->dat.dat[2] >> 32) & 0xffffffff; _buffer[_bufferLen++] = (overlap->dat.dat[2] >> 0) & 0xffffffff; #elif (ovOverlapNWORDS == 8) _buffer[_bufferLen++] = overlap->dat.dat[0]; _buffer[_bufferLen++] = overlap->dat.dat[1]; _buffer[_bufferLen++] = overlap->dat.dat[2]; _buffer[_bufferLen++] = overlap->dat.dat[3]; _buffer[_bufferLen++] = overlap->dat.dat[4]; _buffer[_bufferLen++] = overlap->dat.dat[5]; _buffer[_bufferLen++] = overlap->dat.dat[6]; _buffer[_bufferLen++] = overlap->dat.dat[7]; #else #error unknown ovOverlapNWORDS #endif assert(_bufferLen <= _bufferMax); } void ovFile::writeOverlaps(ovOverlap *overlaps, uint64 overlapsLen) { uint64 nWritten = 0; assert(_isOutput == true); // Resize the olapsPerRead array once per batch. if (_olapsPerRead) { uint32 newmax = _olapsPerReadAlloc; uint32 newlast = _olapsPerReadLast; for (uint32 oo=0; oo= _bufferMax) { AS_UTL_safeWrite(_file, _buffer, "ovFile::writeOverlap", sizeof(uint32), _bufferLen); _bufferLen = 0; } if (_olapsPerRead) { _olapsPerRead[overlaps[nWritten].a_iid]++; _olapsPerRead[overlaps[nWritten].b_iid]++; } if (_isNormal == false) _buffer[_bufferLen++] = overlaps[nWritten].a_iid; _buffer[_bufferLen++] = overlaps[nWritten].b_iid; #if (ovOverlapNWORDS == 5) _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[0]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[1]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[2]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[3]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[4]; #elif (ovOverlapNWORDS == 3) _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[0] >> 32) & 0xffffffff; _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[0] >> 0) & 0xffffffff; _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[1] >> 32) & 0xffffffff; _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[1] >> 0) & 0xffffffff; _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[2] >> 32) & 0xffffffff; _buffer[_bufferLen++] = (overlaps[nWritten].dat.dat[2] >> 0) & 0xffffffff; #elif (ovOverlapNWORDS == 8) _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[0]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[1]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[2]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[3]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[4]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[5]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[6]; _buffer[_bufferLen++] = overlaps[nWritten].dat.dat[7]; #else #error unknown ovOverlapNWORDS #endif nWritten++; } assert(_bufferLen <= _bufferMax); } bool ovFile::readOverlap(ovOverlap *overlap) { assert(_isOutput == false); if (_bufferPos >= _bufferLen) { _bufferLen = AS_UTL_safeRead(_file, _buffer, "ovFile::readOverlap", sizeof(uint32), _bufferMax); _bufferPos = 0; } if (_bufferLen == 0) return(false); assert(_bufferPos < _bufferLen); if (_isNormal == FALSE) overlap->a_iid = _buffer[_bufferPos++]; overlap->b_iid = _buffer[_bufferPos++]; #if (ovOverlapNWORDS == 5) overlap->dat.dat[0] = _buffer[_bufferPos++]; overlap->dat.dat[1] = _buffer[_bufferPos++]; overlap->dat.dat[2] = _buffer[_bufferPos++]; overlap->dat.dat[3] = _buffer[_bufferPos++]; overlap->dat.dat[4] = _buffer[_bufferPos++]; #elif (ovOverlapNWORDS == 3) overlap->dat.dat[0] = _buffer[_bufferPos++]; overlap->dat.dat[0] <<= 32; overlap->dat.dat[0] |= _buffer[_bufferPos++]; overlap->dat.dat[1] = _buffer[_bufferPos++]; overlap->dat.dat[1] <<= 32; overlap->dat.dat[1] |= _buffer[_bufferPos++]; overlap->dat.dat[2] = _buffer[_bufferPos++]; overlap->dat.dat[2] <<= 32; overlap->dat.dat[2] |= _buffer[_bufferPos++]; #elif (ovOverlapNWORDS == 8) overlap->dat.dat[0] = _buffer[_bufferPos++]; overlap->dat.dat[1] = _buffer[_bufferPos++]; overlap->dat.dat[2] = _buffer[_bufferPos++]; overlap->dat.dat[3] = _buffer[_bufferPos++]; overlap->dat.dat[4] = _buffer[_bufferPos++]; overlap->dat.dat[5] = _buffer[_bufferPos++]; overlap->dat.dat[6] = _buffer[_bufferPos++]; overlap->dat.dat[7] = _buffer[_bufferPos++]; #else #error unknown ovOverlapNWORDS #endif assert(_bufferPos <= _bufferLen); return(true); } uint64 ovFile::readOverlaps(ovOverlap *overlaps, uint64 overlapsLen) { uint64 nLoaded = 0; assert(_isOutput == false); while (nLoaded < overlapsLen) { if (_bufferPos >= _bufferLen) { _bufferLen = AS_UTL_safeRead(_file, _buffer, "ovFile::readOverlaps", sizeof(uint32), _bufferMax); _bufferPos = 0; } if (_bufferLen == 0) return(nLoaded); assert(_bufferPos < _bufferLen); if (_isNormal == FALSE) overlaps[nLoaded].a_iid = _buffer[_bufferPos++]; overlaps[nLoaded].b_iid = _buffer[_bufferPos++]; #if (ovOverlapNWORDS == 5) overlaps[nLoaded].dat.dat[0] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[1] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[2] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[3] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[4] = _buffer[_bufferPos++]; #elif (ovOverlapNWORDS == 3) overlaps[nLoaded].dat.dat[0] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[0] <<= 32; overlaps[nLoaded].dat.dat[0] |= _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[1] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[1] <<= 32; overlaps[nLoaded].dat.dat[1] |= _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[2] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[2] <<= 32; overlaps[nLoaded].dat.dat[2] |= _buffer[_bufferPos++]; #elif (ovOverlapNWORDS == 8) overlaps[nLoaded].dat.dat[0] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[1] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[2] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[3] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[4] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[5] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[6] = _buffer[_bufferPos++]; overlaps[nLoaded].dat.dat[7] = _buffer[_bufferPos++]; #else #error unknown ovOverlapNWORDS #endif nLoaded++; assert(_bufferPos <= _bufferLen); } return(nLoaded); } // Move to the correct spot, and force a load on the next readOverlap by setting the position to // the end of the buffer. void ovFile::seekOverlap(off_t overlap) { if (_isSeekable == false) fprintf(stderr, "ovFile::seekOverlap()-- can't seek.\n"), exit(1); AS_UTL_fseek(_file, overlap * recordSize(), SEEK_SET); _bufferPos = _bufferLen; // We probably need to reload the buffer. } canu-1.3/src/stores/ovStoreIndexer.C000066400000000000000000000146021272335422300175030ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVS/overlapStoreIndexer.C * * Modifications by: * * Brian P. Walenz from 2012-APR-02 to 2013-AUG-01 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-15 to 2015-SEP-21 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" int main(int argc, char **argv) { char *ovlName = NULL; uint32 maxJob = 0; bool deleteIntermediates = true; bool doExplicitTest = false; bool doFixes = false; char name[FILENAME_MAX]; argc = AS_configure(argc, argv); int err=0; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-O") == 0) { ovlName = argv[++arg]; } else if (strcmp(argv[arg], "-F") == 0) { maxJob = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-f") == 0) { doFixes = true; } else if (strcmp(argv[arg], "-t") == 0) { doExplicitTest = true; ovlName = argv[++arg]; } else if (strcmp(argv[arg], "-nodelete") == 0) { deleteIntermediates = false; } else { fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); } arg++; } if (ovlName == NULL) err++; if ((maxJob == 0) && (doExplicitTest == false)) err++; if (err) { fprintf(stderr, "usage: %s ...\n", argv[0]); fprintf(stderr, " -O x.ovlStore path to overlap store to build the final index for\n"); fprintf(stderr, " -F s number of slices used in bucketizing/sorting\n"); fprintf(stderr, "\n"); fprintf(stderr, " -t x.ovlStore explicitly test a previously constructed index\n"); fprintf(stderr, " -f when testing, also create a new 'idx.fixed' which might\n"); fprintf(stderr, " resolve rare problems\n"); fprintf(stderr, "\n"); fprintf(stderr, " -nodelete do not remove intermediate files when the index is\n"); fprintf(stderr, " successfully created\n"); fprintf(stderr, "\n"); fprintf(stderr, " DANGER DO NOT USE DO NOT USE DO NOT USE DANGER\n"); fprintf(stderr, " DANGER DANGER\n"); fprintf(stderr, " DANGER This command is difficult to run by hand. DANGER\n"); fprintf(stderr, " DANGER Use ovStoreCreate instead. DANGER\n"); fprintf(stderr, " DANGER DANGER\n"); fprintf(stderr, " DANGER DO NOT USE DO NOT USE DO NOT USE DANGER\n"); fprintf(stderr, "\n"); if (ovlName == NULL) fprintf(stderr, "ERROR: No overlap store (-O) supplied.\n"); if ((maxJob == 0) && (doExplicitTest == false)) fprintf(stderr, "ERROR: One of -F (number of slices) or -t (test a store) must be supplied.\n"); exit(1); } // Do the test, and maybe fix things up. if (doExplicitTest == true) { bool passed = testIndex(ovlName, doFixes); exit((passed == true) ? 0 : 1); } // Check that all segments are present. Every segment should have an info file. uint32 cntJob = 0; for (uint32 i=1; i<=maxJob; i++) { uint32 complete = 0; sprintf(name, "%s/%04d", ovlName, i); if (AS_UTL_fileExists(name, FALSE, FALSE) == true) complete++; else fprintf(stderr, "ERROR: Segment "F_U32" data not present (%s)\n", i, name); sprintf(name, "%s/%04d.info", ovlName, i); if (AS_UTL_fileExists(name, FALSE, FALSE) == true) complete++; else fprintf(stderr, "ERROR: Segment "F_U32" info not present (%s)\n", i, name); sprintf(name, "%s/%04d.index", ovlName, i); if (AS_UTL_fileExists(name, FALSE, FALSE) == true) complete++; else fprintf(stderr, "ERROR: Segment "F_U32" index not present (%s)\n", i, name); if (complete == 3) cntJob++; } if (cntJob != maxJob) { fprintf(stderr, "ERROR: Expected "F_U32" segments, only found "F_U32".\n", maxJob, cntJob); exit(1); } // Merge the stuff. mergeInfoFiles(ovlName, maxJob); // Diagnostics. if (testIndex(ovlName, false) == false) { fprintf(stderr, "ERROR: index failed tests.\n"); exit(1); } // Remove intermediates. For the buckets, we keep going until there are 10 in a row not present. // During testing, on a microbe using 2850 buckets, some buckets were empty. if (deleteIntermediates == false) { fprintf(stderr, "\n"); fprintf(stderr, "Not removing intermediate files. Finished.\n"); exit(0); } fprintf(stderr, "\n"); fprintf(stderr, "Removing intermediate files.\n"); // Removing indices is easy, beacuse we know how many there are. for (uint32 i=1; i<=maxJob; i++) { sprintf(name, "%s/%04u.index", ovlName, i); AS_UTL_unlink(name); sprintf(name, "%s/%04u.info", ovlName, i); AS_UTL_unlink(name); } // We don't know how many buckets there are, so we remove until we fail to find ten // buckets in a row. for (uint32 missing=0, i=1; missing<10; i++) { sprintf(name, "%s/bucket%04d", ovlName, i); if (AS_UTL_fileExists(name, TRUE, FALSE) == FALSE) { missing++; continue; } missing = 0; sprintf(name, "%s/bucket%04d/sliceSizes", ovlName, i); AS_UTL_unlink(name); sprintf(name, "%s/bucket%04d", ovlName, i); rmdir(name); } fprintf(stderr, "Finished.\n"); exit(0); } canu-1.3/src/stores/ovStoreIndexer.mk000066400000000000000000000007351272335422300177320ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := ovStoreIndexer SOURCES := ovStoreIndexer.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/ovStoreSorter.C000066400000000000000000000254531272335422300173710ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_OVS/overlapStoreSorter.C * * Modifications by: * * Brian P. Walenz from 2012-APR-02 to 2013-SEP-09 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-15 to 2015-SEP-21 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include #include using namespace std; // This is the size of the datastructure that we're using to store overlaps for sorting. // At present, with ovOverlap, it is over-allocating a pointer that we don't need, but // to make a custom structure, we'd need to duplicate a bunch of code or copy data after // loading and before writing. // // Used in both ovStoreSorter.C and ovStoreBuild.C. // #define ovOverlapSortSize (sizeof(ovOverlap)) int main(int argc, char **argv) { char *storePath = NULL; char *gkpName = NULL; uint32 fileLimit = 512; // Number of 'slices' from bucketizer uint32 fileID = 0; // 'slice' that we are going to be sorting uint32 jobIdxMax = 0; // Number of 'buckets' from bucketizer uint64 maxMemory = UINT64_MAX; bool deleteIntermediateEarly = false; bool deleteIntermediateLate = false; bool forceRun = false; argc = AS_configure(argc, argv); int err=0; int arg=1; while (arg < argc) { if (strcmp(argv[arg], "-O") == 0) { storePath = argv[++arg]; } else if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-F") == 0) { fileLimit = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-job") == 0) { fileID = atoi(argv[++arg]); jobIdxMax = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-M") == 0) { maxMemory = (uint64)ceil(atof(argv[++arg]) * 1024.0 * 1024.0 * 1024.0); } else if (strcmp(argv[arg], "-deleteearly") == 0) { deleteIntermediateEarly = true; } else if (strcmp(argv[arg], "-deletelate") == 0) { deleteIntermediateLate = true; } else if (strcmp(argv[arg], "-force") == 0) { forceRun = true; } else { fprintf(stderr, "ERROR: unknown option '%s'\n", argv[arg]); } arg++; } if (storePath == NULL) err++; if (fileID == 0) err++; if (jobIdxMax == 0) err++; if (err) { fprintf(stderr, "usage: %s ...\n", argv[0]); fprintf(stderr, " -O x.ovlStore path to overlap store to build the final index for\n"); fprintf(stderr, " -G asm.gkpStore path to gkpStore for this assembly\n"); fprintf(stderr, "\n"); fprintf(stderr, " -F s number of slices used in bucketizing/sorting\n"); fprintf(stderr, " -job j m index of this overlap input file, and max number of files\n"); fprintf(stderr, "\n"); fprintf(stderr, " -M m maximum memory to use, in gigabytes\n"); fprintf(stderr, "\n"); fprintf(stderr, " -deleteearly remove intermediates as soon as possible (unsafe)\n"); fprintf(stderr, " -deletelate remove intermediates when outputs exist (safe)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -force force a recompute, even if the output exists\n"); fprintf(stderr, "\n"); fprintf(stderr, " DANGER DO NOT USE DO NOT USE DO NOT USE DANGER\n"); fprintf(stderr, " DANGER DANGER\n"); fprintf(stderr, " DANGER This command is difficult to run by hand. DANGER\n"); fprintf(stderr, " DANGER Use ovStoreCreate instead. DANGER\n"); fprintf(stderr, " DANGER DANGER\n"); fprintf(stderr, " DANGER DO NOT USE DO NOT USE DO NOT USE DANGER\n"); fprintf(stderr, "\n"); if (storePath == NULL) fprintf(stderr, "ERROR: No overlap store (-O) supplied.\n"); if (fileID == 0) fprintf(stderr, "ERROR: no slice number (-F) supplied.\n"); if (jobIdxMax == 0) fprintf(stderr, "ERROR: no max job ID (-job) supplied.\n"); exit(1); } // Check if we're running or done (or crashed), then note that we're running. { char name[FILENAME_MAX]; sprintf(name,"%s/%04d.ovs", storePath, fileID); if ((forceRun == false) && (AS_UTL_fileExists(name, FALSE, FALSE))) fprintf(stderr, "Job "F_U32" is running or finished (remove '%s' or -force to try again).\n", fileID, name), exit(0); errno = 0; FILE *F = fopen(name, "w"); if (errno) fprintf(stderr, "ERROR: Failed to open '%s' for writing: %s\n", name, strerror(errno)), exit(1); fclose(F); } // Get sizes of each bucket, and the final merge uint64 *sliceSizes = new uint64 [fileLimit + 1]; // For each overlap job, number of overlaps per bucket uint64 *bucketSizes = new uint64 [jobIdxMax + 1]; // For each bucket we care about, number of overlaps uint64 totOvl = 0; uint64 ovlsLen = 0; for (uint32 i=0; i<=jobIdxMax; i++) { bucketSizes[i] = 0; char namz[FILENAME_MAX]; char name[FILENAME_MAX]; sprintf(namz, "%s/bucket%04d/slice%03d.gz", storePath, i, fileID); sprintf(name, "%s/bucket%04d/slice%03d", storePath, i, fileID); if ((AS_UTL_fileExists(namz, FALSE, FALSE) == false) && (AS_UTL_fileExists(name, FALSE, FALSE) == false)) // If no file, there are no overlaps. Skip loading the bucketSizes file. // We expect the gz version to exist (that's the default in bucketizer) more frequently, so // be sure to test for existence of that one first. continue; sprintf(name, "%s/bucket%04d/sliceSizes", storePath, i); FILE *F = fopen(name, "r"); if (errno) fprintf(stderr, "ERROR: Failed to open %s: %s\n", name, strerror(errno)), exit(1); uint64 nr = AS_UTL_safeRead(F, sliceSizes, "sliceSizes", sizeof(uint64), fileLimit + 1); fclose(F); if (nr != fileLimit + 1) { fprintf(stderr, "ERROR: short read on '%s'.\n", name); fprintf(stderr, "ERROR: read "F_U64" sizes insteadof "F_U32".\n", nr, fileLimit + 1); } assert(nr == fileLimit + 1); fprintf(stderr, "Found "F_U64" overlaps from '%s'.\n", sliceSizes[fileID], name); bucketSizes[i] = sliceSizes[fileID]; totOvl += sliceSizes[fileID]; } delete [] sliceSizes; sliceSizes = NULL; if (ovOverlapSortSize * totOvl > maxMemory) { fprintf(stderr, "ERROR: Overlaps need %.2f GB memory, but process limited (via -M) to "F_U64" GB.\n", ovOverlapSortSize * totOvl / 1024.0 / 1024.0 / 1024.0, maxMemory >> 30); char name[FILENAME_MAX]; sprintf(name,"%s/%04d.ovs", storePath, fileID); unlink(name); exit(1); } fprintf(stderr, "Overlaps need %.2f GB memory, allowed to use up to (via -M) "F_U64" GB.\n", ovOverlapSortSize * totOvl / 1024.0 / 1024.0 / 1024.0, maxMemory >> 30); ovOverlap *ovls = ovOverlap::allocateOverlaps(NULL, totOvl); // Load all overlaps - we're guaranteed that either 'name.gz' or 'name' exists (we checked above) // or funny business is happening with our files. for (uint32 i=0; i<=jobIdxMax; i++) { if (bucketSizes[i] == 0) continue; char name[FILENAME_MAX]; sprintf(name, "%s/bucket%04d/slice%03d.gz", storePath, i, fileID); if (AS_UTL_fileExists(name, FALSE, FALSE) == false) sprintf(name, "%s/bucket%04d/slice%03d", storePath, i, fileID); if (AS_UTL_fileExists(name, FALSE, FALSE) == false) fprintf(stderr, "ERROR: "F_U64" overlaps claim to exist in bucket '%s', but file not found.\n", bucketSizes[i], name); fprintf(stderr, "Loading "F_U64" overlaps from '%s'.\n", bucketSizes[i], name); ovFile *bof = new ovFile(name, ovFileFull); uint64 num = 0; while (bof->readOverlap(ovls + ovlsLen)) { ovlsLen++; num++; } if (num != bucketSizes[i]) fprintf(stderr, "ERROR: expected "F_U64" overlaps, found "F_U64" overlaps.\n", bucketSizes[i], num); assert(num == bucketSizes[i]); delete bof; } if (ovlsLen != totOvl) fprintf(stderr, "ERROR: read "F_U64" overlaps, expected "F_U64"\n", ovlsLen, totOvl); assert(ovlsLen == totOvl); if (deleteIntermediateEarly) { char name[FILENAME_MAX]; fprintf(stderr, "Removing inputs.\n"); for (uint32 i=0; i<=jobIdxMax; i++) { if (bucketSizes[i] == 0) continue; sprintf(name, "%s/bucket%04d/slice%03d.gz", storePath, i, fileID); AS_UTL_unlink(name); sprintf(name, "%s/bucket%04d/slice%03d", storePath, i, fileID); AS_UTL_unlink(name); } } // Sort the overlaps - at least on FreeBSD 8.2 with gcc46, the parallel STL sort // algorithms are NOT inplace. Restrict to sequential sorting. // // This sort takes at most 2 minutes on 7gb of overlaps. // fprintf(stderr, "Sorting.\n"); #ifdef _GLIBCXX_PARALLEL // If we have the parallel STL, don't use it! Sort is not inplace! __gnu_sequential::sort(ovls, ovls + ovlsLen); #else sort(ovls, ovls + ovlsLen); #endif // Output to store format fprintf(stderr, "Writing output.\n"); writeOverlaps(storePath, ovls, ovlsLen, fileID); // Clean up. delete [] ovls; if (deleteIntermediateLate) { char name[FILENAME_MAX]; fprintf(stderr, "Removing inputs.\n"); for (uint32 i=0; i<=jobIdxMax; i++) { if (bucketSizes[i] == 0) continue; sprintf(name, "%s/bucket%04d/slice%03d.gz", storePath, i, fileID); AS_UTL_unlink(name); sprintf(name, "%s/bucket%04d/slice%03d", storePath, i, fileID); AS_UTL_unlink(name); } } delete [] bucketSizes; // Remove the sentinel to show we're done. The output is in "%s/%04d". { char name[FILENAME_MAX]; sprintf(name,"%s/%04d.ovs", storePath, fileID); unlink(name); } // Success! return(0); } canu-1.3/src/stores/ovStoreSorter.mk000066400000000000000000000007331272335422300176100ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := ovStoreSorter SOURCES := ovStoreSorter.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/ovStoreStats.C000066400000000000000000000610511272335422300172030ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2015-OCT-27 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-MAR-31 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "ovStore.H" #include "stddev.H" #include "intervalList.H" #define OVL_5 0x01 #define OVL_3 0x02 #define OVL_CONTAINED 0x04 #define OVL_CONTAINER 0x08 #define OVL_PARTIAL 0x10 // Should count unique-contained and repeat-contained separately from unique and repeat // uniq-anchor is also 'plausible chimera' // no-5-prime includes things that entirely cover the read, just no overhang int main(int argc, char **argv) { char *gkpName = NULL; char *ovlName = NULL; char *outPrefix = NULL; uint32 bgnID = 0; uint32 endID = UINT32_MAX; uint32 ovlSelect = 0; double ovlAtMost = AS_OVS_encodeEvalue(1.0); double ovlAtLeast = AS_OVS_encodeEvalue(0.0); double expectedMean = 30.0; double expectedStdDev = 7.0; bool toFile = true; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) gkpName = argv[++arg]; else if (strcmp(argv[arg], "-O") == 0) ovlName = argv[++arg]; else if (strcmp(argv[arg], "-o") == 0) outPrefix = argv[++arg]; else if (strcmp(argv[arg], "-C") == 0) { expectedMean = atof(argv[++arg]); expectedStdDev = atof(argv[++arg]); } else if (strcmp(argv[arg], "-c") == 0) toFile = false; else if (strcmp(argv[arg], "-b") == 0) bgnID = atoi(argv[++arg]); else if (strcmp(argv[arg], "-e") == 0) endID = atoi(argv[++arg]); else if (strcmp(argv[arg], "-overlap") == 0) { arg++; if (strcmp(argv[arg], "5") == 0) ovlSelect |= OVL_5; else if (strcmp(argv[arg], "3") == 0) ovlSelect |= OVL_3; else if (strcmp(argv[arg], "contained") == 0) ovlSelect |= OVL_CONTAINED; else if (strcmp(argv[arg], "container") == 0) ovlSelect |= OVL_CONTAINER; else if (strcmp(argv[arg], "partial") == 0) ovlSelect |= OVL_PARTIAL; else if (strcmp(argv[arg], "atmost") == 0) ovlAtMost = atof(argv[++arg]); else if (strcmp(argv[arg], "atleast") == 0) ovlAtLeast = atof(argv[++arg]); else { fprintf(stderr, "ERROR: unknown -overlap '%s'\n", argv[arg]); exit(1); } } else { fprintf(stderr, "%s: unknown option '%s'.\n", argv[0], argv[arg]); err++; } arg++; } if (gkpName == NULL) err++; if (ovlName == NULL) err++; if (outPrefix == NULL) err++; if (err) { fprintf(stderr, "usage: %s -G gkpStore -O ovlStore -o outPrefix [-b bgnID] [-e endID] ...\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, "Generates statistics for an overlap store. By default all possible classes\n"); fprintf(stderr, "are generated, options can disable specific classes.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -C mean stddev Expect coverage at mean +- stddev\n"); fprintf(stderr, " -c Write stats to stdout, not to a file\n"); fprintf(stderr, "\n"); fprintf(stderr, "Outputs:\n"); fprintf(stderr, "\n"); fprintf(stderr, " outPrefix.per-read.log One line per read, giving readID, read length and classification.\n"); fprintf(stderr, " outPrefix.summary The primary statistical output.\n"); fprintf(stderr, "\n"); fprintf(stderr, "Overlap Selection:\n"); fprintf(stderr, " -overlap 5 5' overlaps only\n"); fprintf(stderr, " -overlap 3 3' overlaps only\n"); fprintf(stderr, " -overlap contained contained overlaps only\n"); fprintf(stderr, " -overlap container container overlaps only\n"); fprintf(stderr, " -overlap partial overlap is not valid for assembly\n"); fprintf(stderr, "\n"); fprintf(stderr, " An overlap is classified as exactly one of 5', 3', contained or container.\n"); fprintf(stderr, " By default, all overlaps are selected. Specifying any of these options will\n"); fprintf(stderr, " restrict overlaps to just those classifications. E.g., '-overlap 5 -overlap 3'\n"); fprintf(stderr, " will select dovetail overlaps off either end of the read.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -overlap atmost x at most fraction x error (overlap-erate <= x)\n"); fprintf(stderr, " -overlap atleast x at least fraction x error (x <= overlap-erate)\n"); fprintf(stderr, "\n"); fprintf(stderr, " Overlaps can be further filtered by fraction error. Usually, this will be an\n"); fprintf(stderr, " 'atmost' filtering to use only the higher qualtiy overlaps.\n"); fprintf(stderr, "\n"); fprintf(stderr, " A contained read has at least one container overlap. Container read -> ---------------\n"); fprintf(stderr, " A container read has at least one contained overlap. Contained overlap -> -----\n"); fprintf(stderr, "\n"); exit(1); } // Set the default to 'all' if nothing set. if (ovlSelect == 0) ovlSelect = 0xff; // Open inputs, find limits. gkStore *gkpStore = gkStore::gkStore_open(gkpName); ovStore *ovlStore = new ovStore(ovlName, gkpStore); if (endID > gkpStore->gkStore_getNumReads()) endID = gkpStore->gkStore_getNumReads(); if (endID < bgnID) fprintf(stderr, "ERROR: invalid bgn/end range bgn=%u end=%u; only %u reads in the store\n", bgnID, endID, gkpStore->gkStore_getNumReads()), exit(1); ovlStore->setRange(bgnID, endID); // Allocate output histograms. histogramStatistics *readNoOlaps = new histogramStatistics; // Bad reads! (read length) histogramStatistics *readHole = new histogramStatistics; histogramStatistics *readHump = new histogramStatistics; histogramStatistics *readNo5 = new histogramStatistics; histogramStatistics *readNo3 = new histogramStatistics; histogramStatistics *olapHole = new histogramStatistics; // Hole size (sum of holes if more than one) histogramStatistics *olapHump = new histogramStatistics; // Hump size (sum of humps if more than one) histogramStatistics *olapNo5 = new histogramStatistics; // 5' uncovered size histogramStatistics *olapNo3 = new histogramStatistics; // 3' uncovered size histogramStatistics *readLowCov = new histogramStatistics; // Good reads! (read length) histogramStatistics *readUnique = new histogramStatistics; histogramStatistics *readRepeatCont = new histogramStatistics; histogramStatistics *readRepeatDove = new histogramStatistics; histogramStatistics *readSpanRepeat = new histogramStatistics; histogramStatistics *readUniqRepeatCont = new histogramStatistics; histogramStatistics *readUniqRepeatDove = new histogramStatistics; histogramStatistics *readUniqAnchor = new histogramStatistics; histogramStatistics *covrLowCov = new histogramStatistics; // Good reads! (overlap length) histogramStatistics *covrUnique = new histogramStatistics; histogramStatistics *covrRepeatCont = new histogramStatistics; histogramStatistics *covrRepeatDove = new histogramStatistics; histogramStatistics *covrSpanRepeat = new histogramStatistics; histogramStatistics *covrUniqRepeatCont = new histogramStatistics; histogramStatistics *covrUniqRepeatDove = new histogramStatistics; histogramStatistics *covrUniqAnchor = new histogramStatistics; histogramStatistics *olapLowCov = new histogramStatistics; // Good reads! (overlap length) histogramStatistics *olapUnique = new histogramStatistics; histogramStatistics *olapRepeatCont = new histogramStatistics; histogramStatistics *olapRepeatDove = new histogramStatistics; histogramStatistics *olapSpanRepeat = new histogramStatistics; histogramStatistics *olapUniqRepeatCont = new histogramStatistics; histogramStatistics *olapUniqRepeatDove = new histogramStatistics; histogramStatistics *olapUniqAnchor = new histogramStatistics; // Coverage interval lists, of all overlaps selected. // Open outputs. char N[FILENAME_MAX]; sprintf(N, "%s.per-read.log", outPrefix); FILE *LOG = fopen(N, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", N, strerror(errno)), exit(1); // Compute! uint32 overlapsMax = 1024; uint32 overlapsLen = 0; ovOverlap *overlaps = ovOverlap::allocateOverlaps(gkpStore, overlapsMax); overlapsLen = ovlStore->readOverlaps(overlaps, overlapsMax); while (overlapsLen > 0) { uint32 readID = overlaps[0].a_iid; uint32 readLen = gkpStore->gkStore_getRead(readID)->gkRead_sequenceLength(); intervalList cov; uint32 covID = 0; bool readCoverage5 = false; bool readCoverage3 = false; bool readContained = false; bool readContainer = false; bool readPartial = false; for (uint32 oo=0; oo ovlAtMost) continue; readCoverage5 |= is5prime; // If there is a 5' overlap, the read isn't missing 5' coverage readCoverage3 |= is3prime; readContained |= isContained; // Read is contained in something else readContainer |= isContainer; // Read is a container of somethign else readPartial |= isPartial; cov.add(overlaps[oo].a_bgn(), overlaps[oo].a_end() - overlaps[oo].a_bgn()); } // If we filtered all the overlaps, just get out of here. Yeah, some code duplication, // but cleaner than sticking an if block around the rest of the loop. if (cov.numberOfIntervals() == 0) { readNoOlaps->add(readLen); overlapsLen = ovlStore->readOverlaps(overlaps, overlapsMax); continue; } // Generate a depth-of-coverage map, then merge intervals intervalList depth(cov); cov.merge(); // Analyze the intervals, save per-read information to the log. uint32 lastInt = cov.numberOfIntervals() - 1; uint32 bgn = cov.lo(0); uint32 end = cov.hi(lastInt); bool contiguous = (lastInt == 0) ? true : false; bool readFullCoverage = (lastInt == 0) && (bgn == 0) && (end == readLen); bool readMissingMiddle = (lastInt != 0); uint32 holeSize = 0; uint32 no5Size = bgn; uint32 no3Size = readLen - end; for (uint32 ii=1; iiadd(readLen); olapHole->add(holeSize); overlapsLen = ovlStore->readOverlaps(overlaps, overlapsMax); continue; } if ((readCoverage5 == false) && (readCoverage3 == false) && (readContained == false) && (readPartial == false)) { fprintf(LOG, "%u\t%u\t%s\n", readID, readLen, "middle-only"); readHump->add(readLen); olapHump->add(no5Size + no3Size); overlapsLen = ovlStore->readOverlaps(overlaps, overlapsMax); continue; } if ((readCoverage5 == false) && (readContained == false) && (readPartial == false)) { fprintf(LOG, "%u\t%u\t%s\n", readID, readLen, "no-5-prime"); readNo5->add(readLen); olapNo5->add(no5Size); overlapsLen = ovlStore->readOverlaps(overlaps, overlapsMax); continue; } if ((readCoverage3 == false) && (readContained == false) && (readPartial == false)) { fprintf(LOG, "%u\t%u\t%s\n", readID, readLen, "no-3-prime"); readNo3->add(readLen); olapNo3->add(no3Size); overlapsLen = ovlStore->readOverlaps(overlaps, overlapsMax); continue; } // Handle good cases. For partial overlaps, bgn and end are not the extent of the read. if (readPartial == false) { assert(bgn == 0); assert(end == readLen); assert(contiguous == true); assert(readFullCoverage == true); } // Compute mean and std.dev of coverage. From this, we decide if the read is 'unique', // 'repeat' or 'mixed'. If 'mixed', we then need to decide if the read spans a repeat, or // joins unique and repeat. double covMean = 0; double covStdDev = 0; for (uint32 ii=0; iiadd(readLen); for (uint32 ii=0; iiadd(depth.depth(ii), depth.hi(ii) - depth.lo(ii)); } if (isUnique) { fprintf(LOG, "%u\t%u\t%s\n", readID, readLen, "unique"); readUnique->add(readLen); for (uint32 ii=0; iiadd(depth.depth(ii), depth.hi(ii) - depth.lo(ii)); } if ((isRepeat) && (readContained == true)) { fprintf(LOG, "%u\t%u\t%s\n", readID, readLen, "contained-repeat"); readRepeatCont->add(readLen); for (uint32 ii=0; iiadd(depth.depth(ii), depth.hi(ii) - depth.lo(ii)); } if ((isRepeat) && (readContained == false)) { fprintf(LOG, "%u\t%u\t%s\n", readID, readLen, "dovetail-repeat"); readRepeatDove->add(readLen); for (uint32 ii=0; iiadd(depth.depth(ii), depth.hi(ii) - depth.lo(ii)); } if (isSpanRepeat) { fprintf(LOG, "%u\t%u\t%s\n", readID, readLen, "span-repeat"); readSpanRepeat->add(readLen); olapSpanRepeat->add(depth.lo(endi) - depth.hi(bgni)); } if ((isUniqRepeat) && (readContained == true)) { fprintf(LOG, "%u\t%u\t%s\n", readID, readLen, "uniq-repeat-cont"); readUniqRepeatCont->add(readLen); } if ((isUniqRepeat) && (readContained == false)) { fprintf(LOG, "%u\t%u\t%s\n", readID, readLen, "uniq-repeat-dove"); readUniqRepeatDove->add(readLen); } if (isUniqAnchor) { fprintf(LOG, "%u\t%u\t%s\n", readID, readLen, "uniq-anchor"); readUniqAnchor->add(readLen); olapUniqAnchor->add(depth.lo(endi) - depth.hi(bgni)); } // Done. Read more data. overlapsLen = ovlStore->readOverlaps(overlaps, overlapsMax); } fclose(LOG); // Done with logging. readHole->finalizeData(); olapHole->finalizeData(); readHump->finalizeData(); olapHump->finalizeData(); readNo5->finalizeData(); olapNo5->finalizeData(); readNo3->finalizeData(); olapNo3->finalizeData(); readLowCov->finalizeData(); olapLowCov->finalizeData(); covrLowCov->finalizeData(); readUnique->finalizeData(); olapUnique->finalizeData(); covrUnique->finalizeData(); readRepeatCont->finalizeData(); olapRepeatCont->finalizeData(); covrRepeatCont->finalizeData(); readRepeatDove->finalizeData(); olapRepeatDove->finalizeData(); covrRepeatDove->finalizeData(); readSpanRepeat->finalizeData(); olapSpanRepeat->finalizeData(); readUniqRepeatCont->finalizeData(); olapUniqRepeatCont->finalizeData(); readUniqRepeatDove->finalizeData(); olapUniqRepeatDove->finalizeData(); readUniqAnchor->finalizeData(); olapUniqAnchor->finalizeData(); LOG = stdout; if (toFile == true) { sprintf(N, "%s.summary", outPrefix); LOG = fopen(N, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", N, strerror(errno)), exit(1); } fprintf(LOG, "category reads %% read length feature size or coverage analysis\n"); fprintf(LOG, "---------------- ------- ------- ---------------------- ------------------------ --------------------\n"); fprintf(LOG, "middle-missing %7"F_U64P" %6.2f %10.2f +- %-8.2f %10.2f +- %-8.2f (bad trimming)\n", readHole->numberOfObjects(), (float)readHole->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readHole->mean(), readHole->stddev(), olapHole->mean(), olapHole->stddev()); fprintf(LOG, "middle-hump %7"F_U64P" %6.2f %10.2f +- %-8.2f %10.2f +- %-8.2f (bad trimming)\n", readHump->numberOfObjects(), (float)readHump->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readHump->mean(), readHump->stddev(), olapHump->mean(), olapHump->stddev()); fprintf(LOG, "no-5-prime %7"F_U64P" %6.2f %10.2f +- %-8.2f %10.2f +- %-8.2f (bad trimming)\n", readNo5->numberOfObjects(), (float)readNo5->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readNo5->mean(), readNo5->stddev(), olapNo5->mean(), olapNo5->stddev()); fprintf(LOG, "no-3-prime %7"F_U64P" %6.2f %10.2f +- %-8.2f %10.2f +- %-8.2f (bad trimming)\n", readNo3->numberOfObjects(), (float)readNo3->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readNo3->mean(), readNo3->stddev(), olapNo3->mean(), olapNo3->stddev()); fprintf(LOG, "\n"); fprintf(LOG, "low-coverage %7"F_U64P" %6.2f %10.2f +- %-8.2f %10.2f +- %-8.2f (easy to assemble, potential for lower quality consensus)\n", readLowCov->numberOfObjects(), (float)readLowCov->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readLowCov->mean(), readLowCov->stddev(), covrLowCov->mean(), covrLowCov->stddev()); fprintf(LOG, "unique %7"F_U64P" %6.2f %10.2f +- %-8.2f %10.2f +- %-8.2f (easy to assemble, perfect, yay)\n", readUnique->numberOfObjects(), (float)readUnique->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readUnique->mean(), readUnique->stddev(), covrUnique->mean(), covrUnique->stddev()); fprintf(LOG, "repeat-cont %7"F_U64P" %6.2f %10.2f +- %-8.2f %10.2f +- %-8.2f (potential for consensus errors, no impact on assembly)\n", readRepeatCont->numberOfObjects(), (float)readRepeatCont->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readRepeatCont->mean(), readRepeatCont->stddev(), covrRepeatCont->mean(), covrRepeatCont->stddev()); fprintf(LOG, "repeat-dove %7"F_U64P" %6.2f %10.2f +- %-8.2f %10.2f +- %-8.2f (hard to assemble, likely won't assemble correctly or even at all)\n", readRepeatDove->numberOfObjects(), (float)readRepeatDove->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readRepeatDove->mean(), readRepeatDove->stddev(), covrRepeatDove->mean(), covrRepeatDove->stddev()); fprintf(LOG, "\n"); fprintf(LOG, "span-repeat %7"F_U64P" %6.2f %10.2f +- %-8.2f %10.2f +- %-8.2f (read spans a large repeat, usually easy to assemble)\n", readSpanRepeat->numberOfObjects(), (float)readSpanRepeat->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readSpanRepeat->mean(), readSpanRepeat->stddev(), olapSpanRepeat->mean(), olapSpanRepeat->stddev()); fprintf(LOG, "uniq-repeat-cont %7"F_U64P" %6.2f %10.2f +- %-8.2f (should be uniquely placed, low potential for consensus errors, no impact on assembly)\n", readUniqRepeatCont->numberOfObjects(), (float)readUniqRepeatCont->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readUniqRepeatCont->mean(), readUniqRepeatCont->stddev()); fprintf(LOG, "uniq-repeat-dove %7"F_U64P" %6.2f %10.2f +- %-8.2f (will end contigs, potential to misassemble)\n", readUniqRepeatDove->numberOfObjects(), (float)readUniqRepeatDove->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readUniqRepeatDove->mean(), readUniqRepeatDove->stddev()); fprintf(LOG, "uniq-anchor %7"F_U64P" %6.2f %10.2f +- %-8.2f %10.2f +- %-8.2f (repeat read, with unique section, probable bad read)\n", readUniqAnchor->numberOfObjects(), (float)readUniqAnchor->numberOfObjects()/gkpStore->gkStore_getNumReads()*100, readUniqAnchor->mean(), readUniqAnchor->stddev(), olapUniqAnchor->mean(), olapUniqAnchor->stddev()); if (toFile == true) fclose(LOG); // Clean up the histograms delete readNoOlaps; delete readHole; delete readHump; delete readNo5; delete readNo3; delete olapHole; delete olapHump; delete olapNo5; delete olapNo3; delete readLowCov; delete readUnique; delete readRepeatCont; delete readRepeatDove; delete readSpanRepeat; delete readUniqRepeatCont; delete readUniqRepeatDove; delete readUniqAnchor; delete covrLowCov; delete covrUnique; delete covrRepeatCont; delete covrRepeatDove; delete covrSpanRepeat; delete covrUniqRepeatCont; delete covrUniqRepeatDove; delete covrUniqAnchor; delete olapLowCov; delete olapUnique; delete olapRepeatCont; delete olapRepeatDove; delete olapSpanRepeat; delete olapUniqRepeatCont; delete olapUniqRepeatDove; delete olapUniqAnchor; delete ovlStore; gkpStore->gkStore_close(); exit(0); } canu-1.3/src/stores/ovStoreStats.mk000066400000000000000000000007311272335422300174260ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := ovStoreStats SOURCES := ovStoreStats.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/tgStore.C000066400000000000000000000475731272335422300161670ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignStore.C * * Modifications by: * * Brian P. Walenz from 2009-OCT-05 to 2014-MAR-31 * are Copyright 2009-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2010-FEB-05 * are Copyright 2010 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-22 to 2015-AUG-11 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "AS_UTL_fileIO.H" #include "tgStore.H" uint32 MASRmagic = 0x5253414d; // 'MASR', as a big endian integer uint32 MASRversion = 1; #define MAX_VERS 1024 // Linked to 10 bits in the header file. tgStore::tgStore(const char *path_, uint32 version_, tgStoreType type_) { // Handle goofy default parameters. These let us retain the previous behavior (before tgStoreType): // new tgStore("path") - to create a new store // new tgStore("path", v) - to open an existing store // // And still allow // new tgStore("path", v, tgStoreCreate) - create new store, make v the current version if (version_ == 0) { version_ = 1; type_ = tgStoreCreate; } // Initialize the object. _type = type_; strcpy(_path, path_); _newTigs = false; _currentVersion = version_; _originalVersion = version_; _tigMax = 0; _tigLen = 0; _tigEntry = NULL; _tigCache = NULL; _dataFile = new dataFileT [MAX_VERS]; for (uint32 i=0; isvID); // The atEOF flag allows us to skip a seek when we're already (supposed) to be at the EOF. This // (hopefully) fixes a problem on one system where the seek() was placing the FP just before EOF // (almost like the last block of data wasn't being flushed), and the tell() would then place the // next tig in the middle of the previous one. // // It also should (greatly) improve performance over NFS, espeically during BOG and CNS. Both of // these only write data, so no repositioning of the stream is needed. // if (_dataFile[te->svID].atEOF == false) { AS_UTL_fseek(FP, 0, SEEK_END); _dataFile[te->svID].atEOF = true; } te->flushNeeded = 0; te->fileOffset = AS_UTL_ftell(FP); //fprintf(stderr, "tgStore::writeTigToDisk()-- write tig "F_S32" in store version "F_U64" at file position "F_U64"\n", // tig->_tigID, te->svID, te->fileOffset); tig->saveToStream(FP); } void tgStore::insertTig(tgTig *tig, bool keepInCache) { // Check that the components do not exceed the bound. // if (tig->_gappedLen > 0) { uint32 len = tig->_gappedLen; uint32 swp = 0; uint32 neg = 0; uint32 pos = 0; for (uint32 i=0; i_childrenLen; i++) { tgPosition *read = tig->_children + i; if ((read->_max < read->_min)) fprintf(stderr, "tgStore::insertTig()-- ERROR: tig %d read %d at (%d,%d) has swapped min/max coordinates\n", tig->_tigID, read->_objID, read->_min, read->_max), swp++; // Could fix, but we currently just fail. This is an algorithmic problem that should be fixed. if ((read->_min < 0) || (read->_max < 0)) fprintf(stderr, "tgStore::insertTig()-- WARNING: tig %d read %d at (%d,%d) has negative position\n", tig->_tigID, read->_objID, read->_min, read->_max), neg++; if (read->_min < 0) read->_min = 0; if (read->_max < 0) read->_max = 0; if ((read->_min > len) || (read->_max > len)) fprintf(stderr, "tgStore::insertTig()-- WARNING: tig %d read %d at (%d,%d) exceeded multialign length %d\n", tig->_tigID, read->_objID, read->_min, read->_max, len), pos++; if (read->_min > len) read->_min = len; if (read->_max > len) read->_max = len; } #if 0 if (swp + neg + pos > 0) { tig->dumpLayout(stderr); fprintf(stderr, "tgStore::insertTig()-- ERROR: tig %d has invalid layout, exceeds bounds of consensus sequence (length %d) -- neg=%d pos=%d -- swp=%d.\n", tig->_tigID, len, neg, pos, swp); } #endif assert(swp == 0); //assert(neg == 0); //assert(pos == 0); } if (tig->_tigID == UINT32_MAX) { tig->_tigID = _tigLen; _newTigs = true; fprintf(stderr, "tgStore::insertTig()-- Added new tig %d\n", tig->_tigID); } if (_tigMax <= tig->_tigID) { while (_tigMax <= tig->_tigID) _tigMax = (_tigMax == 0) ? (1024) : (2 * _tigMax); assert(tig->_tigID < _tigMax); tgStoreEntry *nr = new tgStoreEntry [_tigMax]; tgTig **nc = new tgTig * [_tigMax]; memcpy(nr, _tigEntry, sizeof(tgStoreEntry) * _tigLen); memcpy(nc, _tigCache, sizeof(tgTig *) * _tigLen); memset(nr + _tigLen, 0, sizeof(tgStoreEntry) * (_tigMax - _tigLen)); memset(nc + _tigLen, 0, sizeof(tgTig *) * (_tigMax - _tigLen)); for (uint32 xx=_tigLen; xx<_tigMax; xx++) { nr[xx].isDeleted = true; // Deleted until it gets added, otherwise we try to load and fail. nc[xx] = NULL; } delete [] _tigEntry; delete [] _tigCache; _tigEntry = nr; _tigCache = nc; } _tigLen = MAX(_tigLen, tig->_tigID + 1); _tigEntry[tig->_tigID].tigRecord = *tig; _tigEntry[tig->_tigID].unusedFlags = 0; _tigEntry[tig->_tigID].flushNeeded = true; // Mark as needing a flush by default _tigEntry[tig->_tigID].isDeleted = false; // Now really here! _tigEntry[tig->_tigID].svID = _currentVersion; _tigEntry[tig->_tigID].fileOffset = 123456789; // Write to disk RIGHT NOW unless we're keeping it in cache. If it is written, the flushNeeded // flag is cleared. // if ((keepInCache == false) && (_type != tgStoreReadOnly)) writeTigToDisk(tig, _tigEntry + tig->_tigID); // If the cache is different from this tig, delete the cache. Not sure why this happens -- // did we copy a tig, muck with it, and then want to replace the one in the store? // if ((_tigCache[tig->_tigID] != tig) && (_tigCache[tig->_tigID] != NULL)) { delete _tigCache[tig->_tigID]; _tigCache[tig->_tigID] = NULL; } // Cache it if requested, otherwise clear the cache. // _tigCache[tig->_tigID] = (keepInCache) ? tig : NULL; } void tgStore::deleteTig(uint32 tigID) { assert(tigID < _tigLen); flushDisk(tigID); assert(_tigEntry[tigID].flushNeeded == 0); assert(_tigEntry[tigID].isDeleted == 0); _tigEntry[tigID].isDeleted = 1; delete [] _tigCache[tigID]; _tigCache[tigID] = NULL; } tgTig * tgStore::loadTig(uint32 tigID) { bool cantLoad = true; if (_tigLen <= tigID) fprintf(stderr, "tgStore::loadTig()-- WARNING: invalid out-of-range tigID "F_S32", only "F_S32" ma in store; return NULL.\n", tigID, _tigLen); assert(tigID < _tigLen); //fprintf(stderr, "tgStore::loadTig()-- Loading tig %u (out of %u) from version %u at offest %lu\n", // tigID, _tigLen, // _tigEntry[tigID].svID, // _tigEntry[tigID].fileOffset); // This is...and is not...an error. It does indicate something didn't go according to plan, like // loading a tig that doesn't exist (that should be caught by the above 'tigID < _tigLen' // assert). if (_tigEntry[tigID].isDeleted == true) return(NULL); // This _is_ an error. If a tig is in version zero, it isn't in the store at all. // Someone did something stupid when adding tigs. if (_tigEntry[tigID].svID == 0) return(NULL); // Otherwise, we can load something. if (_tigCache[tigID] == NULL) { FILE *FP = openDB(_tigEntry[tigID].svID); // Since the tig isn't in the cache, it had better NOT be marked as needing to be flushed! assert(_tigEntry[tigID].flushNeeded == false); // Seek to the correct position, and reset the atEOF to indicate we're (with high probability) // not at EOF anymore. if (_dataFile[_tigEntry[tigID].svID].atEOF == true) { fflush(FP); _dataFile[_tigEntry[tigID].svID].atEOF = false; } AS_UTL_fseek(FP, _tigEntry[tigID].fileOffset, SEEK_SET); _tigCache[tigID] = new tgTig; _tigCache[tigID]->loadFromStream(FP); // ALWAYS assume the incore record is more up to date *_tigCache[tigID] = _tigEntry[tigID].tigRecord; // Since we just loaded, no flush is needed. _tigEntry[tigID].flushNeeded = 0; } return(_tigCache[tigID]); } void tgStore::unloadTig(uint32 tigID, bool discardChanges) { if (discardChanges) _tigEntry[tigID].flushNeeded = 0; flushDisk(tigID); assert(_tigEntry[tigID].flushNeeded == 0); delete _tigCache[tigID]; _tigCache[tigID] = NULL; } void tgStore::copyTig(uint32 tigID, tgTig *tigcopy) { assert(tigID < _tigLen); // Deleted? Clear it and return. if (_tigEntry[tigID].isDeleted) { tigcopy->clear(); return; } // In the cache? Deep copy it and return. if (_tigCache[tigID]) { *tigcopy = *_tigCache[tigID]; return; } // Otherwise, load from disk. FILE *FP = openDB(_tigEntry[tigID].svID); // Seek to the correct position, and reset the atEOF to indicate we're (with high probability) // not at EOF anymore. if (_dataFile[_tigEntry[tigID].svID].atEOF == true) { fflush(FP); _dataFile[_tigEntry[tigID].svID].atEOF = false; } AS_UTL_fseek(FP, _tigEntry[tigID].fileOffset, SEEK_SET); tigcopy->clear(); tigcopy->loadFromStream(FP); // ALWAYS assume the incore record is more up to date *tigcopy = _tigEntry[tigID].tigRecord; } void tgStore::flushDisk(uint32 tigID) { if (_tigEntry[tigID].flushNeeded == 0) return; writeTigToDisk(_tigCache[tigID], _tigEntry+tigID); } void tgStore::flushDisk(void) { for (uint32 tigID=0; tigID<_tigLen; tigID++) if ((_tigCache[tigID]) && (_tigEntry[tigID].flushNeeded)) flushDisk(tigID); } void tgStore::flushCache(void) { flushDisk(); for (uint32 i=0; i<_tigLen; i++) if (_tigCache[i]) { delete _tigCache[i]; _tigCache[i] = NULL; } } uint32 tgStore::numTigsInMASRfile(char *name) { uint32 MASRmagicInFile = 0; uint32 MASRversionInFile = 0; uint32 MASRtotalInFile = 0; uint32 indxLen = 0; uint32 masrLen = 0; if (AS_UTL_fileExists(name, false, false) == false) return(0); errno = 0; FILE *F = fopen(name, "r"); if (errno) fprintf(stderr, "tgStore::numTigsInMASRfile()-- Failed to open '%s': %s\n", name, strerror(errno)), exit(1); AS_UTL_safeRead(F, &MASRmagicInFile, "MASRmagic", sizeof(uint32), 1); AS_UTL_safeRead(F, &MASRversionInFile, "MASRversion", sizeof(uint32), 1); AS_UTL_safeRead(F, &MASRtotalInFile, "MASRtotal", sizeof(uint32), 1); fclose(F); if (MASRmagicInFile != MASRmagic) { fprintf(stderr, "tgStore::numTigsInMASRfile()-- Failed to open '%s': magic number mismatch; file=0x%08x code=0x%08x\n", name, MASRmagicInFile, MASRmagic); exit(1); } if (MASRversionInFile != MASRversion) { fprintf(stderr, "tgStore::numTigsInMASRfile()-- Failed to open '%s': version number mismatch; file=%d code=%d\n", name, MASRversionInFile, MASRversion); exit(1); } return(MASRtotalInFile); } void tgStore::dumpMASR(tgStoreEntry* &R, uint32& L, uint32 V) { sprintf(_name, "%s/seqDB.v%03d.tig", _path, V); errno = 0; FILE *F = fopen(_name, "w"); if (errno) fprintf(stderr, "tgStore::dumpMASR()-- Failed to create '%s': %s\n", _name, strerror(errno)), exit(1); AS_UTL_safeWrite(F, &MASRmagic, "MASRmagic", sizeof(uint32), 1); AS_UTL_safeWrite(F, &MASRversion, "MASRversion", sizeof(uint32), 1); AS_UTL_safeWrite(F, &L, "MASRtotal", sizeof(uint32), 1); uint32 indxLen = 0; //fprintf(stderr, "tgStore::dumpMASR()-- Writing '%s' (indxLen=%d masrLen=%d).\n", _name, indxLen, L); // The max isn't written. On load, max is set to length. AS_UTL_safeWrite(F, &indxLen, "MASRindexLen", sizeof(uint32), 1); AS_UTL_safeWrite(F, &L, "MASRlen", sizeof(uint32), 1); AS_UTL_safeWrite(F, R, "MASR", sizeof(tgStoreEntry), L); fclose(F); } void tgStore::loadMASR(tgStoreEntry* &R, uint32& L, uint32& M, uint32 V) { // Allocate space for the data. Search for the latest version, ask it how many tigs are in the // store. // // We don't always need to do this, sometimes we're called to update the data with newer data. // if (R == NULL) { for (int32 i=V; i>0; i--) { sprintf(_name, "%s/seqDB.v%03d.tig", _path, i); L = numTigsInMASRfile(_name); if (L > 0) break; } M = L + 1024; R = new tgStoreEntry [M]; memset(R, 0, sizeof(tgStoreEntry) * M); } sprintf(_name, "%s/seqDB.v%03d.tig", _path, V); while ((AS_UTL_fileExists(_name) == false) && (V > 0)) { V--; sprintf(_name, "%s/seqDB.v%03d.tig", _path, V); } if (V == 0) fprintf(stderr, "tgStore::loadMASR()-- Failed to find any tigs in store '%s'.\n", _path), exit(1); errno = 0; FILE *F = fopen(_name, "r"); if (errno) fprintf(stderr, "tgStore::loadMASR()-- Failed to open '%s': %s\n", _name, strerror(errno)), exit(1); uint32 MASRmagicInFile = 0; uint32 MASRversionInFile = 0; uint32 MASRtotalInFile = 0; uint32 indxLen = 0; uint32 masrLen = 0; AS_UTL_safeRead(F, &MASRmagicInFile, "MASRmagic", sizeof(uint32), 1); AS_UTL_safeRead(F, &MASRversionInFile, "MASRversion", sizeof(uint32), 1); AS_UTL_safeRead(F, &MASRtotalInFile, "MASRtotal", sizeof(uint32), 1); AS_UTL_safeRead(F, &indxLen, "MASRindxLen", sizeof(uint32), 1); AS_UTL_safeRead(F, &masrLen, "MASRmasrLen", sizeof(uint32), 1); if (MASRmagicInFile != MASRmagic) { fprintf(stderr, "tgStore::loadMASR()-- Failed to open '%s': magic number mismatch; file=0x%08x code=0x%08x\n", _name, MASRmagicInFile, MASRmagic); exit(1); } if (MASRversionInFile != MASRversion) { fprintf(stderr, "tgStore::loadMASR()-- Failed to open '%s': version number mismatch; file=%d code=%d\n", _name, MASRversionInFile, MASRversion); exit(1); } // Check we're consistent. if (L < MASRtotalInFile) fprintf(stderr, "tgStore::loadMASR()-- '%s' has more tigs ("F_U32") than expected ("F_U32").\n", _name, MASRtotalInFile, L), exit(1); AS_UTL_safeRead(F, R, "MASR", sizeof(tgStoreEntry), masrLen); fclose(F); } FILE * tgStore::openDB(uint32 version) { if (_dataFile[version].FP) return(_dataFile[version].FP); // Load the data sprintf(_name, "%s/seqDB.v%03d.dat", _path, version); // If version is the _currentVersion, open for writing if allowed. // // "a+" technically writes (always) to the end of file, but this hasn't been tested. errno = 0; if ((_type != tgStoreReadOnly) && (version == _currentVersion)) { _dataFile[version].FP = fopen(_name, "a+"); _dataFile[version].atEOF = false; } else { _dataFile[version].FP = fopen(_name, "r"); _dataFile[version].atEOF = false; } if (errno) fprintf(stderr, "tgStore::openDB()-- Failed to open '%s': %s\n", _name, strerror(errno)), exit(1); return(_dataFile[version].FP); } canu-1.3/src/stores/tgStore.H000066400000000000000000000204421272335422300161560ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignStore.H * src/AS_CNS/MultiAlignStore.h * * Modifications by: * * Brian P. Walenz from 2009-OCT-05 to 2014-MAR-31 * are Copyright 2009-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-22 to 2015-AUG-11 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-29 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef TGSTORE_H #define TGSTORE_H #include "AS_global.H" #include "tgTig.H" // // The tgStore is a disk-resident (with memory cache) database of tgTig structures. // // There are two basic modes of operation: // open a store for reading version v // open a store for reading version v, and writing to version v+1, erasing v+1 before starting // open a store for reading version v, and writing to version v+1, preserving the contents // open a store for reading version v, and writing to version v, preserving the contents // enum tgStoreType { // writable inplace append tgStoreCreate = 0, // Make a new one, then become tgStoreWrite tgStoreReadOnly = 1, // false * * - open version v for reading; inplace=append=false in the code tgStoreWrite = 2, // true false false - open version v+1 for writing, purge contents of v+1; standard open for writing tgStoreAppend = 3, // true false true - open version v+1 for writing, do not purge contents tgStoreModify = 4, // true true false - open version v for writing, do not purge contents }; class tgStore { public: tgStore(const char *path, uint32 version = 0, tgStoreType type = tgStoreReadOnly); ~tgStore(); // Update to the next version. // void nextVersion(void); // Add or update a MA in the store. If keepInCache, we keep a pointer to the tgTig. THE // STORE NOW OWNS THE OBJECT. // void insertTig(tgTig *ma, bool keepInCache); // delete() removes the tig from the cache, and marks it as deleted in the store. // void deleteTig(uint32 tigID); // load() will load and cache the MA. THE STORE OWNS THIS OBJECT. // copy() will load and copy the MA. It will not cache. YOU OWN THIS OBJECT. // tgTig *loadTig(uint32 tigID); void unloadTig(uint32 tigID, bool discardChanges=false); void copyTig(uint32 tigID, tgTig *ma); // Flush to disk any cached MAs. This is called by flushCache(). // void flushDisk(uint32 tigID); void flushDisk(void); // Flush the cache of loaded MAs. Be aware that this is expensive in that the flushed things // usually just get loaded back into core. // void flushCache(uint32 tigID, bool discard=false) { unloadTig(tigID, discard); }; void flushCache(void); uint32 numTigs(void) { return(_tigLen); }; // Accessors to tig data; these do not load the tig from disk. bool isDeleted(uint32 tigID); double getCoverageStat(uint32 tigID); double getMicroHetProb(uint32 tigID); tgTig_class getClass(uint32 tigID); bool getSuggestRepeat(uint32 tigID); bool getSuggestCircular(uint32 tigID); uint32 getNumChildren(uint32 tigID); void setCoverageStat(uint32 tigID, double cs); void setMicroHetProb(uint32 tigID, double mp); void setClass(uint32 tigID, tgTig_class c); void setSuggestRepeat(uint32 tigID, bool enable=true); void setSuggestCircular(uint32 tigID, bool enable=true); uint32 getVersion(uint32 tigID); private: struct tgStoreEntry { tgTigRecord tigRecord; uint64 unusedFlags : 12; // One whole bit for future use. uint64 flushNeeded : 1; // If true, this MAR and associated tig are NOT saved to disk. uint64 isDeleted : 1; // If true, this MAR has been deleted from the assembly. uint64 svID : 10; // 10 -> 1024 versions (HARDCODED in tgStore.C) uint64 fileOffset : 40; // 40 -> 1 TB file size; offset in file where MA is stored }; void writeTigToDisk(tgTig *ma, tgStoreEntry *maRecord); uint32 numTigsInMASRfile(char *name); void dumpMASR(tgStoreEntry* &R, uint32& L, uint32 V); void loadMASR(tgStoreEntry* &R, uint32& L, uint32& M, uint32 V); void purgeVersion(uint32 version); void purgeCurrentVersion(void); friend void operationCompress(char *tigName, int tigVers); FILE *openDB(uint32 V); char _path[FILENAME_MAX]; // Path to the store. char _name[FILENAME_MAX]; // Name of the currently opened file, and other uses. tgStoreType _type; bool _newTigs; // internal flag, set if tigs were added uint32 _originalVersion; // Version we started from (see newTigs in code) uint32 _currentVersion; // Version we are writing to uint32 _tigMax; uint32 _tigLen; tgStoreEntry *_tigEntry; tgTig **_tigCache; struct dataFileT { FILE *FP; bool atEOF; }; dataFileT *_dataFile; // dataFile[version] }; inline bool tgStore::isDeleted(uint32 tigID) { return(_tigEntry[tigID].isDeleted); } inline double tgStore::getCoverageStat(uint32 tigID) { assert(tigID < _tigLen); return(_tigEntry[tigID].tigRecord._coverageStat); } inline double tgStore::getMicroHetProb(uint32 tigID) { assert(tigID < _tigLen); return(_tigEntry[tigID].tigRecord._microhetProb); } inline tgTig_class tgStore::getClass(uint32 tigID) { assert(tigID < _tigLen); return(_tigEntry[tigID].tigRecord._class); } inline bool tgStore::getSuggestRepeat(uint32 tigID) { assert(tigID < _tigLen); return(_tigEntry[tigID].tigRecord._suggestRepeat); } inline bool tgStore::getSuggestCircular(uint32 tigID) { assert(tigID < _tigLen); return(_tigEntry[tigID].tigRecord._suggestCircular); } inline uint32 tgStore::getNumChildren(uint32 tigID) { return(_tigEntry[tigID].tigRecord._childrenLen); } inline void tgStore::setCoverageStat(uint32 tigID, double cs) { assert(tigID < _tigLen); _tigEntry[tigID].tigRecord._coverageStat = cs; if (_tigCache[tigID]) _tigCache[tigID]->_coverageStat = cs; } inline void tgStore::setMicroHetProb(uint32 tigID, double mp) { assert(tigID < _tigLen); _tigEntry[tigID].tigRecord._microhetProb = mp; if (_tigCache[tigID]) _tigCache[tigID]->_microhetProb = mp; } inline void tgStore::setClass(uint32 tigID, tgTig_class c) { assert(tigID < _tigLen); _tigEntry[tigID].tigRecord._class = c; if (_tigCache[tigID]) _tigCache[tigID]->_class = c; } inline void tgStore::setSuggestRepeat(uint32 tigID, bool enable) { assert(tigID < _tigLen); _tigEntry[tigID].tigRecord._suggestRepeat = enable; if (_tigCache[tigID]) _tigCache[tigID]->_suggestRepeat = enable; } inline void tgStore::setSuggestCircular(uint32 tigID, bool enable) { assert(tigID < _tigLen); _tigEntry[tigID].tigRecord._suggestCircular = enable; if (_tigCache[tigID]) _tigCache[tigID]->_suggestCircular = enable; } inline uint32 tgStore::getVersion(uint32 tigID) { assert(tigID < _tigLen); return(_tigEntry[tigID].svID); } #endif canu-1.3/src/stores/tgStoreCompress.C000066400000000000000000000112571272335422300176710ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/stores/tgStore.C * * Modifications by: * * Brian P. Walenz beginning on 2016-APR-18 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" void operationCompress(char *tigName, int tigVers) { tgStore *tigStore = new tgStore(tigName, tigVers); uint32 nErrors = 0; uint32 nCompress = 0; // Fail if this isn't the latest version. If we try to compress something that isn't the latest // version, versions after this still point to the uncompressed tigs. // // Function never written - is this still a problem? (18 APR 2018) // Check that we aren't going to pull a tig out of the future and place it in the past. for (uint32 ti=0; tinumTigs(); ti++) { if (tigStore->isDeleted(ti)) continue; if (tigStore->getVersion(ti) > tigVers) { fprintf(stderr, "WARNING: Attempt to move future unitig "F_U32" from version "F_U32" to previous version %d.\n", ti, tigStore->getVersion(ti), tigVers); nErrors++; } else if (tigStore->getVersion(ti) < tigVers) { nCompress++; } } if (nErrors > 0) { fprintf(stderr, "Store can't be compressed; probably trying to compress to something that isn't the latest version.\n"); fprintf(stderr, " "F_U32" tigs failed; "F_U32" compressable\n", nErrors, nCompress); delete tigStore; exit(1); } // Actually do the moves. if (nCompress > 0) { delete tigStore; tigStore = new tgStore(tigName, tigVers, tgStoreModify); } if (nCompress > 0) { fprintf(stderr, "Compressing "F_U32" tigs into version %d\n", nCompress, tigVers); for (uint32 ti=0; tinumTigs(); ti++) { if ((ti % 1000000) == 0) fprintf(stderr, "tig %d\n", ti); if (tigStore->isDeleted(ti)) { continue; } if (tigStore->getVersion(ti) == tigVers) continue; tgTig *tig = tigStore->loadTig(ti); if (tig == NULL) continue; tigStore->insertTig(tig, true); tigStore->unloadTig(ti); } } // Clean up the older files. if (nCompress > 0) { for (uint32 version=1; versionpurgeVersion(version); } } // And the newer files. delete tigStore; } int main (int argc, char **argv) { char *gkpName = NULL; char *tigName = NULL; int32 tigVers = -1; vector tigInputs; tgStoreType tigType = tgStoreModify; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { tigName = argv[++arg]; tigVers = atoi(argv[++arg]); } else { fprintf(stderr, "%s: unknown option '%s'\n", argv[0], argv[arg]); err++; } arg++; } if ((err) || (gkpName == NULL) || (tigName == NULL) || (tigInputs.size() == 0)) { fprintf(stderr, "usage: %s -G -T \n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " -G Path to the gatekeeper store\n"); fprintf(stderr, " -T Path to the tigStore and version to add tigs to\n"); fprintf(stderr, "\n"); fprintf(stderr, " Remove store versions before . Data present in versions before \n"); fprintf(stderr, " are copied to version . Files for the earlier versions are removed.\n"); fprintf(stderr, "\n"); fprintf(stderr, " WARNING! This code HAS NOT been tested with canu.\n"); fprintf(stderr, "\n"); if (gkpName == NULL) fprintf(stderr, "ERROR: no gatekeeper store (-G) supplied.\n"); if (tigName == NULL) fprintf(stderr, "ERROR: no tig store (-T) supplied.\n"); exit(1); } operationCompress(tigName, tigVers); exit(0); } canu-1.3/src/stores/tgStoreCompress.mk000066400000000000000000000007371272335422300201170ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := tgStoreCompress SOURCES := tgStoreCompress.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/tgStoreCoverageStat.C000066400000000000000000000437471272335422300204760ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/computeCoverageStat.C * * Modifications by: * * Brian P. Walenz from 2011-DEC-20 to 2013-AUG-01 * are Copyright 2011-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2012-JUL-18 * are Copyright 2012 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2015-AUG-14 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-12 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" #include using namespace std; // This program will recompute the coverage statistic for all unitigs in the tigStore. // It replaces at least four implementations (AS_CGB, AS_BOG, AS_BAT, AS_CGW). // // Notes from the AS_CGB version: // // Rho is the number of bases in the chunk between the first fragment arrival and the last fragment // arrival. It is the sum of the fragment overhangs in the chunk. For intuitive purposes you can // think of it as the length of the chunk minus the length of the last fragment (if that last isn't // contained). Thus a singleton chunk has a rho equal to zero. // // A singleton chunk provides no information as to its local fragment arrival rate. We need at // least two closely spaced fragments that are randomly sampled from the chunk to get a local // estimate of the fragment arrival rate. // // The local arrival rate of fragments in the chunk is: // arrival_rate_local = (nfrag_randomly_sampled_in_chunk - 1) / rho // // The arrival distance of fragments in the chunk is the reciprocal of the last formula: // arrival_distance_local = rho / (nfrag_randomly_sampled_in_chunk - 1) // // Note a problem with this formula is that a singleton chunk has a coverage discriminator // statistic of 0/0. // // The formula for the coverage discriminator statistic for the chunk is: // (arrival_rate_global / arrival_rate_local - ln(2)) * (nfrag_randomly_sampled_in_chunk - 1) // // The division by zero singularity cancels out to give the formula: // (arrival_rate_global * rho - ln(2) * (nfrag_randomly_sampled_in_chunk - 1) // // The coverage discriminator statistic should be positive for single coverage, negative for // multiple coverage, and near zero for indecisive. // // ADJUST_FOR_PARTIAL_EXCESS: The standard statistic gives log likelihood ratio of expected depth // vs twice expected depth; but when enough fragments are present, we can actually test whether // depth exceeds expected even fractionally; in deeply sequenced datasets (e.g. bacterial genomes), // this has been observed for repetitive segments. // double ln2 = 0.69314718055994530941723212145818; double globalArrivalRate = 0.0; bool *isNonRandom = NULL; uint32 *readLength = NULL; bool leniant = false; // No frags -> 1 // One frag -> 1 double computeRho(tgTig *tig) { int32 minBgn = INT32_MAX; int32 maxEnd = INT32_MIN; int32 fwdRho = INT32_MIN; int32 revRho = INT32_MAX; // We compute the two rho's using the first definition above - distance between the first // and last fragment arrival. This changes based on the orientation of the unitig, so we // return the average of those two. for (uint32 i=0; inumberOfChildren(); i++) { tgPosition *pos = tig->getChild(i); minBgn = MIN(minBgn, pos->min()); maxEnd = MAX(maxEnd, pos->max()); fwdRho = MAX(fwdRho, pos->min()); // largest begin coord revRho = MIN(revRho, pos->max()); // smallest end coord } if ((leniant == false) && (minBgn != 0)) { fprintf(stderr, "tig %d doesn't begin at zero. Layout:\n", tig->tigID()); tig->dumpLayout(stderr); } if (leniant == false) assert(minBgn == 0); fwdRho = fwdRho - minBgn; revRho = maxEnd - revRho; assert(fwdRho >= 0); assert(revRho >= 0); // AS_CGB is using the begin of the last fragment as rho return((fwdRho + revRho) / 2.0); } uint32 numRandomFragments(tgTig *tig) { uint32 numRand = 0; for (uint32 ii=0; iinumberOfChildren(); ii++) if (isNonRandom[tig->getChild(ii)->ident()] == false) numRand++; return(numRand); } double getGlobalArrivalRate(tgStore *tigStore, FILE *outSTA, uint64 genomeSize, bool useN50) { double globalRate = 0; double recalRate = 0; double sumRho = 0; int32 arLen = 0; double *ar = NULL; uint32 NF; uint64 totalRandom = 0; uint64 totalNF = 0; int32 BIG_SPAN = 10000; int32 big_spans_in_unitigs = 0; // formerly arMax // Go through all the unitigs to sum rho and unitig arrival frags uint32 *allRho = new uint32 [tigStore->numTigs()]; for (uint32 i=0; inumTigs(); i++) { tgTig *tig = tigStore->loadTig(i); allRho[i] = 0; if (tig == NULL) continue; double rho = computeRho(tig); int32 numRandom = numRandomFragments(tig); tigStore->unloadTig(i); sumRho += rho; big_spans_in_unitigs += (int32) (rho / BIG_SPAN); // Keep integral portion of fraction. totalRandom += numRandom; totalNF += (numRandom == 0) ? (0) : (numRandom - 1); allRho[i] = rho; } // Here is a rough estimate of arrival rate. // Use (number frags)/(unitig span) unless unitig span is zero; then use (reads)/(genome). // Here, (number frags) includes only random reads and omits the last read of each unitig. // Here, (sumRho) is total unitig span omitting last read of each unitig. if (genomeSize > 0) { globalRate = totalRandom / (double)genomeSize; } else { if (sumRho > 0) globalRate = totalNF / sumRho; } fprintf(outSTA, "BASED ON ALL UNITIGS:\n"); fprintf(outSTA, "sumRho: %.0f\n", sumRho); fprintf(outSTA, "totalRandomFrags: "F_U64"\n", totalRandom); fprintf(outSTA, "Supplied genome size "F_U64"\n", genomeSize); fprintf(outSTA, "Computed genome size: %.2f\n", totalRandom / globalRate); fprintf(outSTA, "Calculated Global Arrival rate: %f\n", globalRate); // Stop here and return the rough estimate under some circumstances. // *) If user suppled a genome size, we are done. // *) No unitigs. if (genomeSize > 0 || tigStore->numTigs()==0) { delete [] allRho; return(globalRate); } // Calculate rho N50 double rhoN50 = 0; if (useN50) { uint32 growUntil = sumRho / 2; // half is 50%, needed for N50 uint64 growRho = 0; sort (allRho, allRho+tigStore->numTigs()); for (uint32 i=tigStore->numTigs(); i>0; i--) { // from largest to smallest unitig... rhoN50 = allRho[i-1]; growRho += rhoN50; if (growRho >= growUntil) break; // break when sum of rho > 50% } } delete [] allRho; // Try for a better estimate based on just unitigs larger than N50. if (useN50) { double keepRho = 0; double keepNF = 0; for (uint32 i=0; inumTigs(); i++) { tgTig *tig = tigStore->loadTig(i); if (tig == NULL) continue; double rho = computeRho(tig); if (rho < rhoN50) continue; // keep only rho from unitigs > N50 int32 numRandom = numRandomFragments(tig); keepNF += (numRandom == 0) ? (0) : (numRandom - 1); keepRho += rho; tigStore->unloadTig(i); } fprintf(outSTA, "BASED ON UNITIGS > N50:\n"); fprintf(outSTA, "rho N50: %.0f\n", rhoN50); if (keepRho > 1) { // the cutoff 1 is arbitrary but larger than 0.0f globalRate = keepNF / keepRho; fprintf(outSTA, "sumRho: %.0f\n", keepRho); fprintf(outSTA, "totalRandomFrags: %.0f\n", keepNF); fprintf(outSTA, "Computed genome size: %.2f\n", totalRandom / globalRate); fprintf(outSTA, "Calculated Global Arrival rate: %f\n", globalRate); return (globalRate); } else { fprintf(outSTA, "It did not work to re-estimate using the N50 method.\n"); } } // Recompute based on just big unitigs. Big is 10Kbp. double BIG_THRESHOLD = 0.5; int32 big_spans_in_rho = (int32) (sumRho / BIG_SPAN); fprintf(outSTA, "Size of big spans is %d\n", BIG_SPAN); fprintf(outSTA, "Number of big spans in unitigs is %d\n", big_spans_in_unitigs); fprintf(outSTA, "Number of big spans in sum-of-rho is %d\n", big_spans_in_rho); fprintf(outSTA, "Ratio required for re-estimate is %f\n", BIG_THRESHOLD); if ( (big_spans_in_unitigs / big_spans_in_rho) <= BIG_THRESHOLD) { fprintf(outSTA, "Too few big spans to re-estimate using the big spans method.\n"); return(globalRate); } // The test above is a rewrite of the former version, where arMax=big_spans_in_unitigs... // if (arMax <= sumRho / 20000) ar = new double [big_spans_in_unitigs]; for (uint32 i=0; inumTigs(); i++) { tgTig *tig = tigStore->loadTig(i); if (tig == NULL) continue; double rho = computeRho(tig); if (rho <= BIG_SPAN) continue; int32 numRandom = numRandomFragments(tig); double localArrivalRate = numRandom / rho; uint32 rhoDiv10k = rho / BIG_SPAN; assert(0 < rhoDiv10k); for (uint32 aa=0; aa maxDiff) { maxDiff = ard; maxDiffIdx = idx - 1; break; } } double recalRate = 0; recalRate = ar[arLen * 19 / 20]; recalRate = MIN(recalRate, ar[arLen * 1 / 10] * 2.0); recalRate = MIN(recalRate, ar[arLen * 1 / 2] * 1.25); recalRate = MIN(recalRate, ar[maxDiffIdx]); globalRate = MAX(globalRate, recalRate); tigStore->unloadTig(i); } delete [] ar; fprintf(outSTA, "BASED ON BIG SPANS IN UNITIGS:\n"); fprintf(outSTA, "Computed genome size: %.2f (reestimated)\n", totalRandom / globalRate); fprintf(outSTA, "Calculated Global Arrival rate: %f (reestimated)\n", globalRate); return(globalRate); } int main(int argc, char **argv) { char *gkpName = NULL; char *tigName = NULL; int32 tigVers = -1; int64 genomeSize = 0; char *outPrefix = NULL; FILE *outLOG = NULL; FILE *outSTA = NULL; uint32 bgnID = 0; uint32 endID = 0; bool doUpdate = true; bool use_N50 = true; argc = AS_configure(argc, argv); int err = 0; int arg = 1; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { tigName = argv[++arg]; tigVers = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-s") == 0) { genomeSize = atol(argv[++arg]); } else if (strcmp(argv[arg], "-o") == 0) { outPrefix = argv[++arg]; } else if (strcmp(argv[arg], "-n") == 0) { doUpdate = false; } else if (strcmp(argv[arg], "-u") == 0) { use_N50 = false; } else if (strcmp(argv[arg], "-L") == 0) { leniant = true; } else { err++; } arg++; } if (gkpName == NULL) err++; if (tigName == NULL) err++; if (outPrefix == NULL) err++; if (err) { fprintf(stderr, "usage: %s -G gkpStore -T tigStore version -o output-prefix [-s genomeSize] ...\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " -G Mandatory, path G to a gkpStore directory.\n"); fprintf(stderr, " -T Mandatory, path T to a tigStore, and version V.\n"); fprintf(stderr, " -o Mandatory, prefix for output files.\n"); fprintf(stderr, " -s Optional, assume genome size S.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -n Do not update the tigStore (default = do update).\n"); fprintf(stderr, " -u Do not estimate based on N50 (default = use N50).\n"); fprintf(stderr, "\n"); fprintf(stderr, " -L Be leniant; don't require reads start at position zero.\n"); fprintf(stderr, "\n"); if (gkpName == NULL) fprintf(stderr, "No gatekeeper store (-G option) supplied.\n"); if (tigName == NULL) fprintf(stderr, "No input tigStore (-T option) supplied.\n"); if (outPrefix == NULL) fprintf(stderr, "No output prefix (-o option) supplied.\n"); exit(1); } // Open output files first, so we can fail before getting too far along. { char outName[FILENAME_MAX]; errno = 0; sprintf(outName, "%s.log", outPrefix); outLOG = fopen(outName, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", outName, strerror(errno)), exit(1); sprintf(outName, "%s.stats", outPrefix); outSTA = fopen(outName, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", outName, strerror(errno)), exit(1); } // // Load fragment data // fprintf(stderr, "Opening gkpStore '%s'\n", gkpName); gkStore *gkpStore = gkStore::gkStore_open(gkpName, gkStore_readOnly); fprintf(stderr, "Reading read lengths and randomness for %u reads.\n", gkpStore->gkStore_getNumReads()); isNonRandom = new bool [gkpStore->gkStore_getNumReads() + 1]; readLength = new uint32 [gkpStore->gkStore_getNumReads() + 1]; for (uint32 ii=0; ii<=gkpStore->gkStore_getNumReads(); ii++) { gkRead *read = gkpStore->gkStore_getRead(ii); gkLibrary *libr = gkpStore->gkStore_getLibrary(read->gkRead_libraryID()); isNonRandom[ii] = libr->gkLibrary_isNonRandom(); readLength[ii] = read->gkRead_sequenceLength(); } fprintf(stderr, "Closing gkpStore.\n"); gkpStore->gkStore_close(); gkpStore = NULL; // // Open tigs. Kind of important to do this. // fprintf(stderr, "Opening tigStore '%s'\n", tigName); tgStore *tigStore = new tgStore(tigName, tigVers, tgStoreModify); if (endID == 0) endID = tigStore->numTigs(); // // Compute global arrival rate. This ain't cheap. // fprintf(stderr, "Computing global arrival rate.\n"); double globalRate = getGlobalArrivalRate(tigStore, outSTA, genomeSize, use_N50); // // Compute coverage stat for each unitig, populate histograms, write logging. // // Three histograms were made, one for length, coverage stat and arrival distance. The // histograms included // // columns: sum, cumulative_sum, cumulative_fraction, min, average, max // rows: reads, rs reads, nr reads, bases, rho, arrival, discriminator // // Most of those are not defined or null (nr reads? currently zero), and the histograms // in general aren't useful anymore. They were used to help decide if the genome size // was incorrect, causing too many non-unique unitigs. // // They were removed 13 Aug 2015. fprintf(stderr, "Computing coverage stat for tigs %u-%u.\n", bgnID, endID-1); for (uint32 i=bgnID; iloadTig(i); if (tig == NULL) continue; int32 tigLength = tig->length(true); int32 numFrags = tig->numberOfChildren(); int32 numRandom = numRandomFragments(tig); double rho = computeRho(tig); double covStat = 0.0; double arrDist = 0.0; if (numRandom > 1) arrDist = rho / (numRandom - 1); if ((numRandom > 0) && (globalRate > 0.0)) covStat = (rho * globalRate) - (ln2 * (numRandom - 1)); if (i == bgnID) fprintf(outLOG, " tigID rho covStat arrDist\n"); fprintf(outLOG, "%10u %10.2f %10.2f %10.2f\n", tig->tigID(), rho, covStat, arrDist); #undef ADJUST_FOR_PARTIAL_EXCESS #ifdef ADJUST_FOR_PARTIAL_EXCESS // Straight from AS_CGB_all.h, not refactored if(rho>0&&global_fragment_arrival_rate>0.f){ double lambda = global_fragment_arrival_rate * rho; double zscore = ((number_of_randomly_sampled_fragments_in_chunk -1)-lambda) / sqrt(lambda); double p = .5 - erf(zscore/sqrt2)*.5; if(coverage_statistic>5 && p < .001){ fprintf(outSTA, "Standard unitigger a-stat is %f, but only %e chance of this great an excess of fragments: obs = %d, expect = %g rho = " F_S64 " Will reset a-stat to 1.5\n", coverage_statistic,p, number_of_randomly_sampled_fragments_in_chunk-1, lambda,rho); covStat = 1.5; } } #endif if (doUpdate) tigStore->setCoverageStat(tig->tigID(), covStat); tigStore->unloadTig(tig->tigID()); } fclose(outLOG); delete [] isNonRandom; delete [] readLength; delete tigStore; exit(0); } canu-1.3/src/stores/tgStoreCoverageStat.mk000066400000000000000000000007471272335422300207140ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := tgStoreCoverageStat SOURCES := tgStoreCoverageStat.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/tgStoreDump.C000066400000000000000000001143511272335422300170020ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/tigStore.C * * Modifications by: * * Brian P. Walenz from 2009-OCT-05 to 2014-MAR-31 * are Copyright 2009-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Michael Schatz from 2010-FEB-27 to 2010-MAY-17 * are Copyright 2010 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2014-APR-13 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz from 2014-OCT-09 to 2015-AUG-14 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-12 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2016-FEB-22 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" #include "AS_UTL_decodeRange.H" #include "intervalList.H" #include "tgTigSizeAnalysis.H" #undef DEBUG_IGNORE #define DUMP_UNSET 0 #define DUMP_STATUS 1 #define DUMP_TIGS 2 #define DUMP_CONSENSUS 3 #define DUMP_LAYOUT 4 #define DUMP_MULTIALIGN 5 #define DUMP_SIZES 6 #define DUMP_COVERAGE 7 #define DUMP_DEPTH_HISTOGRAM 8 #define DUMP_THIN_OVERLAP 9 #define DUMP_OVERLAP_HISTOGRAM 10 // positions can be // layout (if no consensusExists() // gapped // ungapped // // // MISSING // masking and splitting consensus based on (low) coverage // coverage // single unitig coverage plot and dump of low coverage regions // all unitig histogram of coverage // reporting gc content (need to do this after consensus, while sequence is still in memory, then save in the tig itself) class tgFilter { public: tgFilter() { tigIDbgn = 0; tigIDend = UINT32_MAX; dumpAllClasses = true; dumpUnassembled = false; dumpBubbles = false; dumpContigs = false; minNreads = 0; maxNreads = UINT32_MAX; minLength = 0; maxLength = UINT32_MAX; minCoverage = 0.0; maxCoverage = DBL_MAX; minGoodCov = 0.0; maxGoodCov = DBL_MAX; IL = NULL; ID = NULL; }; ~tgFilter() { delete IL; delete ID; }; bool ignore(tgTig *tig, bool useGapped) { #ifdef DEBUG_IGNORE bool iI = ignoreID(tig); bool iN = ignoreNreads(tig); bool iL = ignoreLength(tig, useGapped); bool iC = ignoreCoverage(tig, useGapped); bool iS = ignoreClass(tig); fprintf(stderr, "ignore()-- tig %u - ignore id %s Nreads %s length %s coverage %s class %s\n", tig->tigID(), (iI) ? "true" : "false", (iN) ? "true" : "false", (iL) ? "true" : "false", (iC) ? "true" : "false", (iS) ? "true" : "false"); #endif return(ignoreID(tig) || ignoreNreads(tig) || ignoreLength(tig, useGapped) || ignoreCoverage(tig, useGapped) || ignoreClass(tig)); }; bool ignoreID(tgTig *tig) { return((tig->tigID() < tigIDbgn) || (tigIDend < tig->tigID())); }; bool ignoreClass(tgTig *tig) { if ((dumpAllClasses == true) || ((tig->_class == tgTig_unassembled) && (dumpUnassembled == true)) || ((tig->_class == tgTig_bubble) && (dumpBubbles == true)) || ((tig->_class == tgTig_contig) && (dumpContigs == true))) return(false); return(true); }; bool ignoreNreads(tgTig *tig) { return((tig->numberOfChildren() < minNreads) || (maxNreads < tig->numberOfChildren())); }; bool ignoreLength(tgTig *tig, bool useGapped) { uint32 length = tig->length(useGapped); return((length < minLength) || (maxLength < length)); }; bool ignoreCoverage(tgTig *tig, bool useGapped) { if ((minCoverage == 0) && (maxCoverage == UINT32_MAX)) return(false); if (tig->consensusExists() == false) useGapped = true; delete IL; IL = new intervalList; for (uint32 i=0; inumberOfChildren(); i++) { tgPosition *pos = tig->getChild(i); int32 bgn = (useGapped) ? pos->min() : tig->mapGappedToUngapped(pos->min()); int32 end = (useGapped) ? pos->max() : tig->mapGappedToUngapped(pos->max()); IL->add(bgn, end - bgn); } delete ID; ID = new intervalList(*IL); uint32 goodCov = 0; uint32 badCov = 0; for (uint32 ii=0; iinumberOfIntervals(); ii++) if ((minCoverage <= ID->depth(ii)) && (ID->depth(ii) <= maxCoverage)) goodCov += ID->hi(ii) - ID->lo(ii); else badCov += ID->hi(ii) - ID->lo(ii); double fracGood = (goodCov) / (goodCov + badCov); return((fracGood < minGoodCov) || (maxGoodCov < fracGood)); }; void maskConsensus(char *cns) { for (uint32 ii=0; iinumberOfIntervals(); ii++) { if ((minCoverage <= ID->depth(ii)) && (ID->depth(ii) <= maxCoverage)) for (uint32 pp=ID->lo(ii); pphi(ii); pp++) cns[pp] = '_'; } }; uint32 tigIDbgn; uint32 tigIDend; bool dumpAllClasses; bool dumpUnassembled; bool dumpBubbles; bool dumpContigs; uint32 minNreads; uint32 maxNreads; uint32 minLength; uint32 maxLength; double minCoverage; double maxCoverage; double minGoodCov; double maxGoodCov; intervalList *IL; intervalList *ID; }; void dumpStatus(gkStore *UNUSED(gkpStore), tgStore *tigStore) { fprintf(stderr, "%u\n", tigStore->numTigs()); } void dumpTig(FILE *out, tgTig *tig, bool useGapped) { fprintf(out, F_U32"\t"F_U32"\t%s\t%.2f\t%s\t%s\t%s\t"F_U32"\n", tig->tigID(), tig->length(useGapped), tig->coordinateType(useGapped), tig->_coverageStat, toString(tig->_class), tig->_suggestRepeat ? "yes" : "no", tig->_suggestCircular ? "yes" : "no", tig->numberOfChildren()); } void dumpRead(FILE *out, tgTig *tig, tgPosition *read, bool useGapped) { fprintf(out, F_U32"\t"F_U32"\t%s\t"F_U32"\t"F_U32"\n", read->ident(), tig->tigID(), tig->coordinateType(useGapped), (useGapped) ? read->bgn() : tig->mapGappedToUngapped(read->bgn()), (useGapped) ? read->end() : tig->mapGappedToUngapped(read->end())); } void dumpTigs(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool useGapped) { fprintf(stdout, "tigID\ttigLen\ttype\tcovStat\tsr\tsu\tsc\tsh\tnumChildren\n"); for (uint32 ti=0; tinumTigs(); ti++) { if (tigStore->isDeleted(ti)) continue; tgTig *tig = tigStore->loadTig(ti); if (tig->consensusExists() == false) useGapped = true; if (filter.ignore(tig, useGapped) == true) { tigStore->unloadTig(ti); continue; } dumpTig(stdout, tig, useGapped); tigStore->unloadTig(ti); } } void dumpConsensus(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool useGapped, char cnsFormat) { for (uint32 ti=0; tinumTigs(); ti++) { if (tigStore->isDeleted(ti)) continue; tgTig *tig = tigStore->loadTig(ti); if (tig->consensusExists() == false) { //fprintf(stderr, "dumpConsensus()-- tig %u has no consensus sequence.\n", ti); tigStore->unloadTig(ti); continue; } if (filter.ignore(tig, useGapped) == true) { tigStore->unloadTig(ti); continue; } switch (cnsFormat) { case 'A': tig->dumpFASTA(stdout, useGapped); break; case 'Q': tig->dumpFASTQ(stdout, useGapped); break; default: break; } tigStore->unloadTig(ti); } } void dumpLayout(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool useGapped, char *outPrefix) { FILE *tigs = NULL; // Length and flags of tigs, same as dumpTigs() FILE *reads = NULL; // Length and flags of reads, mapping of read to tig FILE *layout = stdout; // Standard layout file if (outPrefix) { char T[FILENAME_MAX]; int32 Terr = 0; char R[FILENAME_MAX]; int32 Rerr = 0; char L[FILENAME_MAX]; int32 Lerr = 0; sprintf(T, "%s.layout.tigInfo", outPrefix); sprintf(R, "%s.layout.readToTig", outPrefix); sprintf(L, "%s.layout", outPrefix); errno = 0; tigs = fopen(T, "w"); Terr = errno; reads = fopen(R, "w"); Rerr = errno; layout = fopen(L, "w"); Lerr = errno; if (Terr) fprintf(stderr, "Failed to open '%s': %s\n", T, strerror(Terr)); if (Rerr) fprintf(stderr, "Failed to open '%s': %s\n", R, strerror(Rerr)); if (Lerr) fprintf(stderr, "Failed to open '%s': %s\n", L, strerror(Lerr)); if (Terr + Rerr + Lerr > 0) exit(1); } for (uint32 ti=0; tinumTigs(); ti++) { if (tigStore->isDeleted(ti)) continue; tgTig *tig = tigStore->loadTig(ti); if (tig->consensusExists() == false) useGapped = true; if (filter.ignore(tig, useGapped) == true) { tigStore->unloadTig(ti); continue; } if (tigs) dumpTig(tigs, tig, useGapped); if (reads) for (uint32 ci=0; cinumberOfChildren(); ci++) dumpRead(reads, tig, tig->getChild(ci), useGapped); if (layout) tig->dumpLayout(layout); tigStore->unloadTig(ti); } if (outPrefix) { fclose(tigs); fclose(reads); fclose(layout); } } void dumpMultialign(gkStore *gkpStore, tgStore *tigStore, tgFilter &filter, bool maWithQV, bool maWithDots, uint32 maDisplayWidth, uint32 maDisplaySpacing) { for (uint32 ti=0; tinumTigs(); ti++) { if (tigStore->isDeleted(ti)) continue; tgTig *tig = tigStore->loadTig(ti); if (filter.ignore(tig, true) == true) { tigStore->unloadTig(ti); continue; } tig->display(stdout, gkpStore, maDisplayWidth, maDisplaySpacing, maWithQV, maWithDots); tigStore->unloadTig(ti); } } void dumpSizes(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool useGapped, uint64 genomeSize) { tgTigSizeAnalysis *siz = new tgTigSizeAnalysis(genomeSize); for (uint32 ti=0; tinumTigs(); ti++) { if (tigStore->isDeleted(ti)) continue; tgTig *tig = tigStore->loadTig(ti); if (tig->consensusExists() == false) useGapped = true; if (filter.ignore(tig, useGapped) == true) { tigStore->unloadTig(ti); continue; } siz->evaluateTig(tig, useGapped); tigStore->unloadTig(ti); } siz->finalize(); siz->printSummary(stdout); delete siz; } void plotDepthHistogram(char *N, uint64 *cov, uint32 covMax) { FILE *F; // Find the smallest and largest values with counts. uint32 minii = 0; uint32 maxii = covMax - 1; while ((minii < covMax) && (cov[minii] == 0)) minii++; while ((minii < maxii) && (cov[maxii] == 0)) maxii--; // Extend a little bit, to give a little context (and to get zero). minii = (minii > 10) ? minii-10 : 0; maxii = maxii+10; // Dump the values from min to max errno = 0; F = fopen(N, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", N, strerror(errno)); for (uint32 ii=minii; ii<=maxii; ii++) for (uint32 xx=0; xx /dev/null 2>&1", "w"); if (F) { fprintf(F, "set terminal 'png'\n"); fprintf(F, "set output '%s.png'\n", N); fprintf(F, "set xlabel 'binwidth=%u'\n", boxsize); //fprintf(F, "set ylabel 'number of bases'\n"); // A histogram, assuming the data is sorted values fprintf(F, "binwidth=%u\n", boxsize); fprintf(F, "set boxwidth binwidth\n"); fprintf(F, "bin(x,width) = width*floor(x/width) + binwidth/2.0\n"); fprintf(F, "plot [%u:%u] '%s' using (bin($1,binwidth)):(1.0) smooth freq with boxes title '%s'\n", minii, maxii, N, N); // A histogram, assuming the data is a histogram. It's ugly though. //fprintf(F, "plot [%u:%u] [] '%s' using 1:2 with lines title '%s', \\\n", // Used to report 'tigs %u-%u' for the range // minii, maxii, N, N); pclose(F); } // Dump the values again, this time as a real histogram. errno = 0; F = fopen(N, "w"); if (errno) fprintf(stderr, "Failed to open '%s' for writing: %s\n", N, strerror(errno)); for (uint32 ii=minii; ii<=maxii; ii++) fprintf(F, F_U32"\t"F_U64"\n", ii, cov[ii]); fclose(F); } void dumpDepthHistogram(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool useGapped, bool single, char *outPrefix) { char N[FILENAME_MAX]; intervalList IL; int32 covMax = 1048576; uint64 *cov = new uint64 [covMax]; memset(cov, 0, sizeof(uint64) * covMax); for (uint32 ti=0; tinumTigs(); ti++) { if (tigStore->isDeleted(ti)) continue; tgTig *tig = tigStore->loadTig(ti); if (tig->consensusExists() == false) useGapped = true; if (filter.ignore(tig, useGapped) == true) { tigStore->unloadTig(ti); continue; } // Save all the read intervals to the list. IL.clear(); for (uint32 ci=0; cinumberOfChildren(); ci++) { tgPosition *read = tig->getChild(ci); uint32 bgn = (useGapped) ? read->min() : tig->mapGappedToUngapped(read->min()); uint32 end = (useGapped) ? read->max() : tig->mapGappedToUngapped(read->max()); IL.add(bgn, end - bgn); } // Convert to depths. intervalList ID(IL); // Add the depths to the histogram. for (uint32 ii=0; iitigID()); plotDepthHistogram(N, cov, covMax); memset(cov, 0, sizeof(uint64) * covMax); // Slight optimization if we do this in plotDepthHistogram of just the set values. } // Repeat. tigStore->unloadTig(ti); } if (single == false) { sprintf(N, "%s.depthHistogram", outPrefix); plotDepthHistogram(N, cov, covMax); } delete cov; } void dumpCoverage(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool useGapped, char *outPrefix) { uint32 covMax = 1024; uint64 *cov = new uint64 [covMax]; for (uint32 ti=0; tinumTigs(); ti++) { if (tigStore->isDeleted(ti)) continue; tgTig *tig = tigStore->loadTig(ti); uint32 tigLen = tig->length(useGapped); if (tig->consensusExists() == false) useGapped = true; if (filter.ignore(tig, true) == true) { tigStore->unloadTig(ti); continue; } // Do something. intervalList allL; for (uint32 ci=0; cinumberOfChildren(); ci++) { tgPosition *read = tig->getChild(ci); uint32 bgn = (useGapped) ? read->min() : tig->mapGappedToUngapped(read->min()); uint32 end = (useGapped) ? read->max() : tig->mapGappedToUngapped(read->max()); allL.add(bgn, end - bgn); } intervalList ID(allL); uint32 maxDepth = 0; double aveDepth = 0; double sdeDepth = 0; #if 0 // Report regions that have abnormally low or abnormally high coverage intervalList minL; intervalList maxL; for (uint32 ii=0; iitigID(), ID.lo(ii), ID.hi(ii), tigLen, ID.depth(ii)); minL.add(ID.lo(ii), ID.hi(ii) - ID.lo(ii) + 1); } if (maxCoverage <= ID.depth(ii)) { fprintf(stderr, "tig %d high coverage interval %ld %ld max %u coverage %u\n", tig->tigID(), ID.lo(ii), ID.hi(ii), tigLen, ID.depth(ii)); maxL.add(ID.lo(ii), ID.hi(ii) - ID.lo(ii) + 1); } } #endif // Compute max and average depth, and save the depth in a histogram. #warning replace this with genericStatistics for (uint32 ii=0; ii maxDepth) maxDepth = ID.depth(ii); aveDepth += (ID.hi(ii) - ID.lo(ii) + 1) * ID.depth(ii); while (covMax <= ID.depth(ii)) resizeArray(cov, covMax, covMax, covMax * 2); cov[ID.depth(ii)] += ID.hi(ii) - ID.lo(ii) + 1; } aveDepth /= tigLen; // Now the std.dev for (uint32 ii=0; ii 0) && (maxL.numberOfIntervals() > 0)) fprintf(stderr, "tig %d has %u intervals, %u regions below %u coverage and %u regions at or above %u coverage\n", tig->tigID(), allL.numberOfIntervals(), minL.numberOfIntervals(), minCoverage, maxL.numberOfIntervals(), maxCoverage); else if (minL.numberOfIntervals() > 0) fprintf(stderr, "tig %d has %u intervals, %u regions below %u coverage\n", tig->tigID(), allL.numberOfIntervals(), minL.numberOfIntervals(), minCoverage); else if (maxL.numberOfIntervals() > 0) fprintf(stderr, "tig %d has %u intervals, %u regions at or above %u coverage\n", tig->tigID(), allL.numberOfIntervals(), maxL.numberOfIntervals(), maxCoverage); else fprintf(stderr, "tig %d has %u intervals\n", tig->tigID(), allL.numberOfIntervals()); #endif // Plot the depth for each tig if (outPrefix) { char outName[FILENAME_MAX]; sprintf(outName, "%s.tig%08u.depth", outPrefix, tig->tigID()); FILE *outFile = fopen(outName, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", outName, strerror(errno)), exit(1); for (uint32 ii=0; ii /dev/null 2>&1", "w"); if (gnuPlot) { fprintf(gnuPlot, "set terminal 'png'\n"); fprintf(gnuPlot, "set output '%s.tig%08u.png'\n", outPrefix, tig->tigID()); fprintf(gnuPlot, "set xlabel 'position'\n"); fprintf(gnuPlot, "set ylabel 'coverage'\n"); fprintf(gnuPlot, "set terminal 'png'\n"); fprintf(gnuPlot, "plot '%s.tig%08u.depth' using 1:2 with lines title 'tig %u length %u', \\\n", outPrefix, tig->tigID(), tig->tigID(), tigLen); fprintf(gnuPlot, " %f title 'mean %.2f +- %.2f', \\\n", aveDepth, aveDepth, sdeDepth); fprintf(gnuPlot, " %f title '' lt 0 lc 2, \\\n", aveDepth - sdeDepth); fprintf(gnuPlot, " %f title '' lt 0 lc 2\n", aveDepth + sdeDepth); fclose(gnuPlot); } } // Did something. tigStore->unloadTig(ti); } } void dumpThinOverlap(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool useGapped, uint32 minOverlap) { fprintf(stderr, "reporting overlaps of at most %u bases\n", minOverlap); for (uint32 ti=0; tinumTigs(); ti++) { if (tigStore->isDeleted(ti)) continue; tgTig *tig = tigStore->loadTig(ti); if (tig->consensusExists() == false) useGapped = true; if (filter.ignore(tig, true) == true) { tigStore->unloadTig(ti); continue; } // Do something. intervalList allL; intervalList ovlL; intervalList badL; for (uint32 ri=0; rinumberOfChildren(); ri++) { tgPosition *read = tig->getChild(ri); uint32 bgn = (useGapped) ? read->min() : tig->mapGappedToUngapped(read->min()); uint32 end = (useGapped) ? read->max() : tig->mapGappedToUngapped(read->max()); allL.add(bgn, end - bgn); ovlL.add(bgn, end - bgn); } allL.merge(); // Merge, requiring zero overlap (adjacent is OK) between pieces ovlL.merge(minOverlap); // Merge, requiring minOverlap overlap between pieces // If there is more than one interval, make a list of the regions where we have thin overlaps. if (ovlL.numberOfIntervals() > 1) // Vertical space between tig reports fprintf(stderr, "\n"); for (uint32 ii=1; iitigID(), ovlL.lo(ii), ovlL.hi(ii-1)); badL.add(ovlL.lo(ii), ovlL.hi(ii-1) - ovlL.lo(ii)); } // Then report any reads that intersect that region. for (uint32 ri=0; rinumberOfChildren(); ri++) { tgPosition *read = tig->getChild(ri); uint32 bgn = (useGapped) ? read->min() : tig->mapGappedToUngapped(read->min()); uint32 end = (useGapped) ? read->max() : tig->mapGappedToUngapped(read->max()); bool report = false; for (uint32 oo=0; ootigID(), read->ident(), (useGapped) ? read->min() : tig->mapGappedToUngapped(read->min()), (useGapped) ? read->max() : tig->mapGappedToUngapped(read->max())); } if ((allL.numberOfIntervals() != 1) || (ovlL.numberOfIntervals() != 1)) fprintf(stderr, "tig %d %s length %u has %u interval%s and %u interval%s after enforcing minimum overlap of %u\n", tig->tigID(), tig->coordinateType(useGapped), tig->length(), allL.numberOfIntervals(), (allL.numberOfIntervals() == 1) ? "" : "s", ovlL.numberOfIntervals(), (ovlL.numberOfIntervals() == 1) ? "" : "s", minOverlap); // There, did something. tigStore->unloadTig(ti); } } void dumpOverlapHistogram(gkStore *UNUSED(gkpStore), tgStore *tigStore, tgFilter &filter, bool useGapped, char *outPrefix) { uint32 histMax = AS_MAX_READLEN; uint64 *hist = new uint64 [histMax]; memset(hist, 0, sizeof(uint64) * histMax); for (uint32 ti=0; tinumTigs(); ti++) { if (tigStore->isDeleted(ti)) continue; tgTig *tig = tigStore->loadTig(ti); int32 tn = tig->numberOfChildren(); if (tig->consensusExists() == false) useGapped = true; if (filter.ignore(tig, true) == true) { tigStore->unloadTig(ti); continue; } // Do something. For each read, compute the thickest overlap off of each end. // First, decide on positions for each read. Store in an array for easier use later. uint32 *bgn = new uint32 [tn]; uint32 *end = new uint32 [tn]; for (uint32 ri=0; rigetChild(ri); bgn[ri] = (useGapped) ? read->min() : tig->mapGappedToUngapped(read->min()); end[ri] = (useGapped) ? read->max() : tig->mapGappedToUngapped(read->max()); } // Scan these, marking contained reads. for (uint32 ri=0; ri bgn[ri] for (int32 ii=ri-1; ii>0; ii--) { if (bgn[ii] == UINT32_MAX) continue; if (end[ii] < bgn[ri]) // Read doesn't overlap, no more reads will. break; if (thickest5 < end[ii] - bgn[ri]) thickest5 = end[ii] - bgn[ri]; } // Off the 3' end, expect bgn[ii] < end[ri] and bgn[ii] > bgn[ri] for (int32 ii=ri+1; ii 0) { assert(thickest5 < histMax); hist[thickest5]++; } if (thickest3 > 0) { assert(thickest3 < histMax); hist[thickest3]++; } } delete [] bgn; delete [] end; // There, did something. tigStore->unloadTig(ti); } // All computed. Dump the data and plot. char N[FILENAME_MAX]; sprintf(N, "%s.thickestOverlapHistogram", outPrefix); plotDepthHistogram(N, hist, histMax); // Cleanup and Bye! delete [] hist; } int main (int argc, char **argv) { char *gkpName = NULL; char *tigName = NULL; int tigVers = -1; // Tig Selection tgFilter filter; // Dump options uint32 dumpType = DUMP_UNSET; bool useGapped = false; char cnsFormat = 'A'; // Or 'Q' for FASTQ bool maWithQV = false; bool maWithDots = true; uint32 maDisplayWidth = 100; uint32 maDisplaySpacing = 3; uint64 genomeSize = 0; char *outPrefix = NULL; bool single = false; uint32 minOverlap = 0; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { tigName = argv[++arg]; tigVers = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-tig") == 0) { AS_UTL_decodeRange(argv[++arg], filter.tigIDbgn, filter.tigIDend); } else if (strcmp(argv[arg], "-unassembled") == 0) { filter.dumpAllClasses = false; filter.dumpUnassembled = true; } else if (strcmp(argv[arg], "-bubbles") == 0) { filter.dumpAllClasses = false; filter.dumpBubbles = true; } else if (strcmp(argv[arg], "-contigs") == 0) { filter.dumpAllClasses = false; filter.dumpContigs = true; } else if (strcmp(argv[arg], "-nreads") == 0) { filter.minNreads = atoi(argv[++arg]); filter.maxNreads = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-length") == 0) { filter.minLength = atoi(argv[++arg]); filter.maxLength = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-coverage") == 0) { if ((arg == argc-1) || (argv[arg+1][0] == '-')) { dumpType = DUMP_COVERAGE; } else if (arg + 4 < argc) { filter.minCoverage = atof(argv[++arg]); filter.maxCoverage = atof(argv[++arg]); filter.minGoodCov = atof(argv[++arg]); filter.maxGoodCov = atof(argv[++arg]); } else { fprintf(stderr, "ERROR: -coverage needs four values.\n"); err++; } } // Dump types. else if (strcmp(argv[arg], "-status") == 0) dumpType = DUMP_STATUS; else if (strcmp(argv[arg], "-tigs") == 0) dumpType = DUMP_TIGS; else if (strcmp(argv[arg], "-consensus") == 0) dumpType = DUMP_CONSENSUS; else if (strcmp(argv[arg], "-layout") == 0) dumpType = DUMP_LAYOUT; else if (strcmp(argv[arg], "-multialign") == 0) dumpType = DUMP_MULTIALIGN; else if (strcmp(argv[arg], "-sizes") == 0) dumpType = DUMP_SIZES; else if (strcmp(argv[arg], "-coverage") == 0) // NOTE! Actually handled above. dumpType = DUMP_COVERAGE; else if (strcmp(argv[arg], "-depth") == 0) dumpType = DUMP_DEPTH_HISTOGRAM; else if (strcmp(argv[arg], "-overlap") == 0) dumpType = DUMP_THIN_OVERLAP; else if (strcmp(argv[arg], "-overlaphistogram") == 0) dumpType = DUMP_OVERLAP_HISTOGRAM; // Options. else if (strcmp(argv[arg], "-gapped") == 0) useGapped = true; else if (strcmp(argv[arg], "-fasta") == 0) cnsFormat = 'A'; else if (strcmp(argv[arg], "-fastq") == 0) cnsFormat = 'Q'; else if (strcmp(argv[arg], "-w") == 0) maDisplayWidth = atoi(argv[++arg]); else if (strcmp(argv[arg], "-s") == 0) maDisplaySpacing = genomeSize = atol(argv[++arg]); else if (strcmp(argv[arg], "-o") == 0) outPrefix = argv[++arg]; else if (strcmp(argv[arg], "-single") == 0) single = true; else if (strcmp(argv[arg], "-thin") == 0) minOverlap = atoi(argv[++arg]); // Errors. else { fprintf(stderr, "%s: Unknown option '%s'\n", argv[0], argv[arg]); err++; } arg++; } if (gkpName == NULL) err++; if (tigName == NULL) err++; if ((outPrefix == NULL) && (dumpType == DUMP_COVERAGE)) err++; if (dumpType == DUMP_UNSET) err++; if (err) { fprintf(stderr, "usage: %s -G -T [opts]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, "STORE SELECTION (mandatory)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -G path to the gatekeeper store\n"); fprintf(stderr, " -T path to the tigStore, version, to use\n"); fprintf(stderr, "\n"); fprintf(stderr, "TIG SELECTION - if nothing specified, all tigs are reported\n"); fprintf(stderr, " - all ranges are inclusive.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -tig A[-B] only dump tigs between ids A and B\n"); fprintf(stderr, " -unassembled only dump tigs that are 'unassembled'\n"); fprintf(stderr, " -bubbles only dump tigs that are 'bubbles'\n"); fprintf(stderr, " -contigs only dump tigs that are 'contigs'\n"); fprintf(stderr, " -nreads min max only dump tigs with between min and max reads\n"); fprintf(stderr, " -length min max only dump tigs with length between 'min' and 'max' bases\n"); fprintf(stderr, " -coverage c C g G only dump tigs with between fraction g and G at coverage between c and C\n"); fprintf(stderr, " example: -coverage 10 inf 0.5 1.0 would report tigs where half of the\n"); fprintf(stderr, " bases are at 10+ times coverage.\n"); fprintf(stderr, "\n"); fprintf(stderr, "DUMP TYPE - all dumps, except status, report on tigs selected as above\n"); fprintf(stderr, "\n"); fprintf(stderr, " -status the number of tigs in the store\n"); fprintf(stderr, "\n"); fprintf(stderr, " -tigs a list of tigs, and some information about them\n"); fprintf(stderr, "\n"); fprintf(stderr, " -consensus [opts] the consensus sequence, with options:\n"); fprintf(stderr, " -gapped report the gapped (multialignment) consensus sequence\n"); fprintf(stderr, " -fasta report sequences in FASTA format (the default)\n"); fprintf(stderr, " -fastq report sequences in FASTQ format\n"); fprintf(stderr, "\n"); fprintf(stderr, " -layout [opts] the layout of reads in each tig\n"); fprintf(stderr, " if '-o' is supplied, three files are created, otherwise just the layout is printed to stdout\n"); fprintf(stderr, " -gapped report the gapped (multialignment) positions\n"); fprintf(stderr, " -o outputPrefix write plots to 'outputPrefix.*' in the current directory\n"); fprintf(stderr, "\n"); fprintf(stderr, " -multialign [opts] the full multialignment, output is to stdout\n"); fprintf(stderr, " -w width width of the page\n"); fprintf(stderr, " -s spacing spacing between reads on the same line\n"); fprintf(stderr, "\n"); fprintf(stderr, " -sizes [opts] size statistics\n"); fprintf(stderr, " -s genomesize denominator to use for n50 computation\n"); fprintf(stderr, "\n"); fprintf(stderr, " -coverage [opts] read coverage plots, one plot per tig\n"); fprintf(stderr, " -o outputPrefix write plots to 'outputPrefix.*' in the current directory\n"); fprintf(stderr, "\n"); fprintf(stderr, " -depth [opts] a histogram of depths\n"); fprintf(stderr, " -single one histogram per tig\n"); fprintf(stderr, "\n"); fprintf(stderr, " -overlap read overlaps\n"); fprintf(stderr, " -thin overlap report regions where the (thickest) read overlap is less than 'overlap' bases\n"); fprintf(stderr, "\n"); fprintf(stderr, " -overlaphistogram a histogram of the thickest overlaps used\n"); fprintf(stderr, " -o outputPrefix write plots to 'outputPrefix.*' in the current directory\n"); fprintf(stderr, "\n"); fprintf(stderr, "\n"); #if 0 fprintf(stderr, " -compress Move tigs from earlier versions into the specified version. This removes\n"); fprintf(stderr, " historical versions of unitigs/contigs, and can save tremendous storage space,\n"); fprintf(stderr, " but makes it impossible to back up the assembly past the specified versions\n"); #endif if (gkpName == NULL) err++; if (tigName == NULL) err++; if ((outPrefix == NULL) && (dumpType == DUMP_COVERAGE)) err++; if (dumpType == DUMP_UNSET) err++; exit(1); } // Open stores. gkStore *gkpStore = gkStore::gkStore_open(gkpName); tgStore *tigStore = new tgStore(tigName, tigVers); // Check that the tig ID range is valid, and fix it if possible. uint32 nTigs = tigStore->numTigs(); if (filter.tigIDend == UINT32_MAX) filter.tigIDend = nTigs-1; if (nTigs <= filter.tigIDend) { fprintf(stderr, "WARNING: adjusting tig ID range from "F_U32"-"F_U32" to "F_U32"-"F_U32" as there are only "F_U32" tigs in the store.\n", filter.tigIDbgn, filter.tigIDend, filter.tigIDbgn, nTigs-1, nTigs); filter.tigIDend = nTigs - 1; } if (filter.tigIDend < filter.tigIDbgn) { fprintf(stderr, "WARNING: adjusting inverted tig ID range -t "F_U32"-"F_U32"\n", filter.tigIDbgn, filter.tigIDend); uint32 x = filter.tigIDend; filter.tigIDend = filter.tigIDbgn; filter.tigIDbgn = x; } if (nTigs <= filter.tigIDbgn) fprintf(stderr, "ERROR: only "F_U32" tigs in the store (IDs 0-"F_U32" inclusive); can't dump requested range -t "F_U32"-"F_U32"\n", nTigs, nTigs-1, filter.tigIDbgn, filter.tigIDend), exit(1); // Call the dump routine. switch (dumpType) { case DUMP_STATUS: dumpStatus(gkpStore, tigStore); break; case DUMP_TIGS: dumpTigs(gkpStore, tigStore, filter, useGapped); break; case DUMP_CONSENSUS: dumpConsensus(gkpStore, tigStore, filter, useGapped, cnsFormat); break; case DUMP_LAYOUT: dumpLayout(gkpStore, tigStore, filter, useGapped, outPrefix); break; case DUMP_MULTIALIGN: dumpMultialign(gkpStore, tigStore, filter, maWithQV, maWithDots, maDisplayWidth, maDisplaySpacing); break; case DUMP_SIZES: dumpSizes(gkpStore, tigStore, filter, useGapped, genomeSize); break; case DUMP_COVERAGE: dumpCoverage(gkpStore, tigStore, filter, useGapped, outPrefix); break; case DUMP_DEPTH_HISTOGRAM: dumpDepthHistogram(gkpStore, tigStore, filter, useGapped, single, outPrefix); break; case DUMP_THIN_OVERLAP: dumpThinOverlap(gkpStore, tigStore, filter, useGapped, minOverlap); break; case DUMP_OVERLAP_HISTOGRAM: dumpOverlapHistogram(gkpStore, tigStore, filter, useGapped, outPrefix); break; default: break; } // Clean up. delete tigStore; gkpStore->gkStore_close(); // Bye. exit(0); } canu-1.3/src/stores/tgStoreDump.mk000066400000000000000000000007711272335422300172270ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := tgStoreDump SOURCES := tgStoreDump.C \ tgTigSizeAnalysis.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/tgStoreFilter.C000066400000000000000000000462201272335422300173210ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_BAT/markRepeatUnique.C * * Modifications by: * * Brian P. Walenz from 2014-MAR-31 to 2014-APR-15 * are Copyright 2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren on 2014-APR-14 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz from 2014-OCT-09 to 2015-AUG-14 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-30 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" #include "intervalList.H" #include using namespace std; // Stats on repeat labeling of input unitigs. // class ruLabelStat { public: ruLabelStat() { num = 0; len = 0; }; void operator+=(uint64 len_) { num++; len += len_; }; uint32 num; uint64 len; }; ruLabelStat repeat_LowReads; ruLabelStat repeat_LowCovStat; ruLabelStat repeat_Short; ruLabelStat repeat_SingleSpan; ruLabelStat repeat_LowCov; ruLabelStat repeat_IsSingleton; ruLabelStat repeat_IsUnique; ruLabelStat repeat_IsRepeat; intervalList * computeCoverage(tgTig *tig) { intervalList IL; for (uint32 ii=0; iinumberOfChildren(); ii++) { tgPosition *pos = tig->getChild(ii); IL.add(pos->min(), pos->max() - pos->min()); } return(new intervalList(IL)); } int main(int argc, char **argv) { char *gkpName = NULL; char *tigName = NULL; int32 tigVers = -1; int64 genomeSize = 0; char outName[FILENAME_MAX]; char *outPrefix = NULL; FILE *outLOG = NULL; FILE *outSTA = NULL; // From the original description of these values (CGB_UNIQUE_CUTOFF): // A threshold value for Gene's coverage statistic. Values above this value have never been // known to associated with unitigs with fragments that are not contiguous in the genome. double cgbUniqueCutoff = 10.0; double cgbDefinitelyUniqueCutoff = 10.0; double singleReadMaxCoverage = 1.0; // Reads covering more than this will demote the unitig uint32 lowCovDepth = 2; double lowCovFractionAllowed = 1.0; uint32 tooLong = UINT32_MAX; uint32 tooShort = 1000; uint32 minReads = 2; double minPopulous = 0; uint32 bgnID = 0; uint32 endID = 0; uint32 maxID = 0; gkStore *gkpStore = NULL; tgStore *tigStore = NULL; tgStoreType tigMode = tgStoreModify; argc = AS_configure(argc, argv); fprintf(stderr, "this is obsolete. do not use.\n"); exit(1); int err = 0; int arg = 1; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { tigName = argv[++arg]; tigVers = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-o") == 0) { outPrefix = argv[++arg]; } else if (strcmp(argv[arg], "-n") == 0) { tigMode = tgStoreReadOnly; } else if (strcmp(argv[arg], "-j") == 0) { cgbUniqueCutoff = atof(argv[++arg]); } else if (strcmp(argv[arg], "-k") == 0) { cgbDefinitelyUniqueCutoff = atof(argv[++arg]); } else if (strcmp(argv[arg], "-span") == 0) { singleReadMaxCoverage = atof(argv[++arg]); // If a single read spans more than this fraction of unitig, it is demoted } else if (strcmp(argv[arg], "-lowcov") == 0) { lowCovDepth = atoi(argv[++arg]); // Coverage below this is too low lowCovFractionAllowed = atof(argv[++arg]); // If unitig has more than this fraction low coverage, it is demoted } else if (strcmp(argv[arg], "-reads") == 0) { arg++; minReads = atoi(argv[arg]); // If unitig has fewer than this number of reads it is demoted minPopulous = atof(argv[arg]); if (minPopulous > 1.0) minPopulous = 0; } else if (strcmp(argv[arg], "-long") == 0) { tooLong = atoi(argv[++arg]); // Unitigs longer than this cannot be demoted } else if (strcmp(argv[arg], "-short") == 0) { tooShort = atoi(argv[++arg]); // Unitigs shorter than this are demoted } else { err++; } arg++; } if (gkpName == NULL) err++; if (tigName == NULL) err++; if (err) { fprintf(stderr, "usage: %s -g gkpStore -t tigStore version\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " -G Mandatory, path G to a gkpStore directory.\n"); fprintf(stderr, " -T Mandatory, path T to a tigStore, and version V.\n"); fprintf(stderr, "\n"); fprintf(stderr, " -j J Unitig is not unique if astat is below J (cgbUniqueCutoff)\n"); fprintf(stderr, " -k K (unused) (cgbDefinitelyUniqueCutoff)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -span F Unitig is not unique if a single read spans more than fraction F (default 1.0) of unitig\n"); fprintf(stderr, " -lowcov D F Unitig is not unique if fraction F (default 1.0) of unitig is below read depth D (default 2)\n"); fprintf(stderr, " -reads R Unitig is not unique if unitig has fewer than R (default 2) reads\n"); fprintf(stderr, " If R is fractional, the least populous unitigs containing fraction R of reads are marked as repeat\n"); fprintf(stderr, " Example: unitigs with 9 or fewer reads contain 10%% of the reads. -reads 0.10 would mark these are repeat.\n"); fprintf(stderr, " -long L Unitig is unique if unitig is at least L (default unlimited) bases long\n"); fprintf(stderr, " -short S Unitig is not unique if unitig is shorter than S (default 1000) bases long\n"); fprintf(stderr, "\n"); fprintf(stderr, " -o Prefix for output files.\n"); fprintf(stderr, " -n Do not update the tigStore.\n"); fprintf(stderr, "\n"); fprintf(stderr, "Algorithm: The first rule to trigger will mark the unitig.\n"); fprintf(stderr, "\n"); fprintf(stderr, " 1) A unitig with a single read is NOT unique.\n"); fprintf(stderr, " 2) A unitig with fewer than R (-reads) reads is NOT unique.\n"); fprintf(stderr, " 3) A unitig with a single read spanning fraction F (-span) of the unitig is NOT unique.\n"); fprintf(stderr, " 4) A unitig longer than L (-length) bases IS unique.\n"); fprintf(stderr, " 5) A unitig with astat less than J (-j) is NOT unique.\n"); fprintf(stderr, " 6) A unitig with fraction F below coverage D (-lowcov) is NOT unique.\n"); fprintf(stderr, " 7) A unitig shorter than S (-short) bases long is NOT unique.\n"); fprintf(stderr, " 8) Otherwise, the unitig IS unique.\n"); if (gkpName == NULL) fprintf(stderr, "No gatekeeper store (-G option) supplied.\n"); if (tigName == NULL) fprintf(stderr, "No input tigStore (-T option) supplied.\n"); if (outPrefix == NULL) fprintf(stderr, "No output prefix (-o option) supplied.\n"); exit(1); } gkpStore = gkStore::gkStore_open(gkpName, gkStore_readOnly); tigStore = new tgStore(tigName, tigVers, tgStoreReadOnly); if (endID == 0) endID = tigStore->numTigs(); maxID = tigStore->numTigs(); errno = 0; sprintf(outName, "%s.log", outPrefix); outLOG = fopen(outName, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", outName, strerror(errno)), exit(1); fprintf(outLOG, "tigID\trho\tcovStat\tarrDist\n"); sprintf(outName, "%s.stats", outPrefix); outSTA = fopen(outName, "w"); if (errno) fprintf(stderr, "Failed to open '%s': %s\n", outName, strerror(errno)), exit(1); fprintf(stderr, "Command Line options:\n"); fprintf(stderr, " singleReadMaxCoverage %f\n", singleReadMaxCoverage); fprintf(stderr, " lowCoverage %u coverage %f fraction\n", lowCovDepth, lowCovFractionAllowed); if (minPopulous > 0) fprintf(stderr, " minReads recomputed based on least populous fraction %.4f\n", minPopulous); else fprintf(stderr, " minReads %u\n", minReads); fprintf(stderr, " tooLong %u\n", tooLong); fprintf(stderr, " tooShort %u\n", tooShort); // // Load fragment data // fprintf(stderr, "Loading fragment data.\n"); double globalRate = 0; bool *isNonRandom = new bool [gkpStore->gkStore_getNumReads() + 1]; uint32 *fragLength = new uint32 [gkpStore->gkStore_getNumReads() + 1]; for (uint32 fi=1; fi <= gkpStore->gkStore_getNumReads(); fi++) { gkRead *read = gkpStore->gkStore_getRead(fi); gkLibrary *libr = gkpStore->gkStore_getLibrary(read->gkRead_libraryID()); isNonRandom[fi] = libr->gkLibrary_isNonRandom(); fragLength[fi] = read->gkRead_sequenceLength(); } // // Compute: // global coverage histogram // global single-read fraction covered - number of bases in unitigs with specific fraction covered // global number of reads per unitig // // This is using GAPPED lengths, because they're faster, and we don't need the actual ungapped positions. fprintf(stderr, "Generating statistics.\n"); uint32 covHistogramMax = 1048576; uint32 *covHistogram = new uint32 [covHistogramMax]; uint32 **utgCovHistogram = new uint32 * [maxID]; uint32 *utgCovData = new uint32 [maxID * lowCovDepth]; memset(covHistogram, 0, sizeof(uint32) * covHistogramMax); memset(utgCovHistogram, 0, sizeof(uint32) * maxID); memset(utgCovData, 0, sizeof(uint32) * maxID * lowCovDepth); uint32 *singleReadCoverageHistogram = new uint32 [1001]; double *singleReadCoverage = new double [maxID]; memset(singleReadCoverageHistogram, 0, sizeof(uint32) * 1001); memset(singleReadCoverage, 0, sizeof(double) * maxID); uint32 numReadsPerUnitigMax = maxID + 1; uint32 *numReadsPerUnitig = new uint32 [numReadsPerUnitigMax]; memset(numReadsPerUnitig, 0, sizeof(uint32) * numReadsPerUnitigMax); for (uint32 uu=0; uuloadTig(uu); uint32 tigLen = tig->length(true); if (tig == NULL) continue; utgCovHistogram[tig->tigID()] = utgCovData + tig->tigID() * lowCovDepth; if (tig->numberOfChildren() == 1) continue; // Global coverage histogram. intervalList *ID = computeCoverage(tig); for (uint32 ii=0; iinumberOfIntervals(); ii++) { if (ID->depth(ii) < lowCovDepth) utgCovHistogram[tig->tigID()][ID->depth(ii)] += ID->hi(ii) - ID->lo(ii) + 1; if (ID->depth(ii) < covHistogramMax) covHistogram[ID->depth(ii)] += ID->hi(ii) - ID->lo(ii) + 1; } delete ID; ID = NULL; // Single read max fraction covered. uint32 covMax = 0; uint32 cov; for (uint32 ff=0; ffnumberOfChildren(); ff++) { tgPosition *pos = tig->getChild(ff); cov = 1000 * (pos->max() - pos->min()) / tigLen; if (covMax < cov) covMax = cov; } singleReadCoverageHistogram[covMax]++; singleReadCoverage[tig->tigID()] = covMax / 1000.0; // Number of reads per unitig numReadsPerUnitig[uu] = tig->numberOfChildren(); //fprintf(stderr, "unitig %u covMax %f\n", tig->tigID(), covMax / 1000.0); tigStore->unloadTig(uu); } // // Analyze our collected data, decide on some thresholds. // fprintf(stderr, "Analyzing statistics.\n"); if ((minPopulous > 0.0) && (minPopulous < 1.0)) { uint32 maxReadsPerUnitig = 0; for (uint32 uu=0; uu 0) fprintf(outSTA, "minReads %u with fraction %.04f of reads\n", hist[xx], vals[xx]); delete [] totReadsPerNumReads; } // // Apply the thresholds to unitigs. The first half of these are the historical CGW rules. // fprintf(stderr, "Processing unitigs %u to %u.\n", bgnID, endID); for (uint32 uu=bgnID; uuloadTig(uu); if (tig == NULL) { fprintf(outLOG, "unitig %d not present\n", uu); continue; } // This uses UNGAPPED lengths, because they make more sense to humans. uint32 tigLen = tig->length(false); uint32 lowCovBases = 0; for (uint32 ll=0; lltigID()][ll]; bool isUnique = true; bool isSingleton = false; if (tig->numberOfChildren() == 1) { fprintf(outLOG, "unitig %d not unique -- singleton\n", tig->tigID()); isUnique = false; isSingleton = true; } else if (tig->numberOfChildren() < minReads) { fprintf(outLOG, "unitig %d not unique -- %u reads, need at least %d\n", tig->tigID(), tig->numberOfChildren(), minReads); repeat_LowReads += tigLen; isUnique = false; } else if (singleReadCoverage[tig->tigID()] > singleReadMaxCoverage) { fprintf(outLOG, "unitig %d not unique -- single read spans fraction %f of unitig (>= %f)\n", tig->tigID(), singleReadCoverage[tig->tigID()], singleReadMaxCoverage); repeat_SingleSpan += tigLen; isUnique = false; } else if (tigLen >= tooLong) { fprintf(outLOG, "unitig %d IS unique -- too long to be repeat, %u > allowed %u\n", tig->tigID(), tigLen, tooLong); isUnique = true; } else if (tigStore->getCoverageStat(tig->tigID()) < cgbUniqueCutoff) { fprintf(outLOG, "unitig %d not unique -- coverage stat %f, needs to be at least %f\n", tig->tigID(), tigStore->getCoverageStat(tig->tigID()), cgbUniqueCutoff); repeat_LowCovStat += tigLen; isUnique = false; } else if ((double)lowCovBases / tigLen > lowCovFractionAllowed) { fprintf(outLOG, "unitig %d not unique -- too many low coverage bases, %u out of %u bases, fraction %f > allowed %f\n", tig->tigID(), lowCovBases, tigLen, (double)lowCovBases / tigLen, lowCovFractionAllowed); repeat_LowCov += tigLen; isUnique = false; } // This was an attempt to not blindly call all short unitigs as non-unique. It didn't work so // well in initial limited testing. The threshold is arbitrary; older versions used // cgbDefinitelyUniqueCutoff. If used, be sure to disable the real check after this! #if 0 else if ((tigStore->getCoverageStat(tig->tigID()) < cgbUniqueCutoff * 10) && (tigLen < CGW_MIN_DISCRIMINATOR_UNIQUE_LENGTH)) { fprintf(outLOG, "unitig %d not unique -- length %d too short, need to be at least %d AND coverage stat %d must be larger than %d\n", tig->tigID(), tigLen, CGW_MIN_DISCRIMINATOR_UNIQUE_LENGTH, tigStore->getCoverageStat(tig->tigID()), cgbUniqueCutoff * 10); repeat_Short += tigLen; isUnique = false; } #endif else if (tigLen < tooShort) { fprintf(outLOG, "unitig %d not unique -- length %d too short, need to be at least %d\n", tig->tigID(), tigLen, tooShort); repeat_Short += tigLen; isUnique = false; } else { fprintf(outLOG, "unitig %d not repeat -- no test failed\n", tig->tigID()); } // // Allow flag to override the rules and force it to be unique or repeat. AKA, toggling. // if (isUnique) { repeat_IsUnique += tigLen; #warning NOT setSuggestUnique //tigStore->setSuggestUnique(tig->tigID()); tigStore->setSuggestRepeat(tig->tigID(), false); } else if (isSingleton) { repeat_IsSingleton += tigLen; #warning NOT setSuggestUnique //tigStore->setSuggestUnique(tig->tigID(), false); tigStore->setSuggestRepeat(tig->tigID()); } else { repeat_IsRepeat += tigLen; #warning NOT setSuggestUnique //tigStore->setSuggestUnique(tig->tigID(), false); tigStore->setSuggestRepeat(tig->tigID()); } } fprintf(outSTA, "classification number of unitigs total length\n"); fprintf(outSTA, " unique: %17"F_U32P" %14"F_U64P"\n", repeat_IsUnique.num, repeat_IsUnique.len); fprintf(outSTA, " singleton: %17"F_U32P" %14"F_U64P"\n", repeat_IsSingleton.num, repeat_IsSingleton.len); fprintf(outSTA, " repeat: %17"F_U32P" %14"F_U64P"\n", repeat_IsRepeat.num, repeat_IsRepeat.len); fprintf(outSTA, " too few reads: %17"F_U32P" %14"F_U64P"\n", repeat_LowReads.num, repeat_LowReads.len); fprintf(outSTA, " low cov stat: %17"F_U32P" %14"F_U64P"\n", repeat_LowCovStat.num, repeat_LowCovStat.len); fprintf(outSTA, " too short: %17"F_U32P" %14"F_U64P"\n", repeat_Short.num, repeat_Short.len); fprintf(outSTA, " spanning read: %17"F_U32P" %14"F_U64P"\n", repeat_SingleSpan.num, repeat_SingleSpan.len); fprintf(outSTA, " low coverage: %17"F_U32P" %14"F_U64P"\n", repeat_LowCov.num, repeat_LowCov.len); fclose(outLOG); fclose(outSTA); delete [] isNonRandom; delete [] fragLength; gkpStore->gkStore_close(); delete tigStore; exit(0); } canu-1.3/src/stores/tgStoreFilter.mk000066400000000000000000000007331272335422300175450ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := tgStoreFilter SOURCES := tgStoreFilter.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/tgStoreLoad.C000066400000000000000000000147101272335422300167520ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-AUG-07 to 2015-AUG-14 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" void operationBuild(char *buildName, char *tigName, uint32 tigVers) { errno = 0; FILE *F = fopen(buildName, "r"); if (errno) fprintf(stderr, "Failed to open '%s' for reading: %s\n", buildName, strerror(errno)), exit(1); if (AS_UTL_fileExists(tigName, TRUE, TRUE)) { fprintf(stderr, "ERROR: '%s' exists, and I will not clobber an existing store.\n", tigName); exit(1); } tgStore *tigStore = new tgStore(tigName); tgTig *tig = new tgTig(); for (int32 v=1; vnextVersion(); while (tig->loadLayout(F) == true) { if (tig->numberOfChildren() == 0) continue; // The log isn't correct. For new tigs (all of these are) we don't know the // id until after it is added. Further, if these come with id's already set, // they can't be added to a new store -- they don't exist. #if 0 fprintf(stderr, "INSERTING tig %d (%d children) (originally ID %d)\n", tig->tigID(), tig->numberOfChildren(), oID); #endif tigStore->insertTig(tig, false); } fclose(F); delete tig; delete tigStore; } int main (int argc, char **argv) { char *gkpName = NULL; char *tigName = NULL; int32 tigVers = -1; vector tigInputs; tgStoreType tigType = tgStoreModify; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { tigName = argv[++arg]; tigVers = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-L") == 0) { AS_UTL_loadFileList(argv[++arg], tigInputs); } else if (strcmp(argv[arg], "-n") == 0) { tigType = tgStoreReadOnly; } else if (AS_UTL_fileExists(argv[arg])) { tigInputs.push_back(argv[arg]); } else { fprintf(stderr, "%s: unknown option '%s'\n", argv[0], argv[arg]); err++; } arg++; } if ((err) || (gkpName == NULL) || (tigName == NULL) || (tigInputs.size() == 0)) { fprintf(stderr, "usage: %s -G -T [input.cns]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " -G Path to the gatekeeper store\n"); fprintf(stderr, " -T Path to the tigStore and version to add tigs to\n"); fprintf(stderr, "\n"); fprintf(stderr, " -L Load the tig(s) from files listed in 'file-of-files'\n"); fprintf(stderr, "\n"); fprintf(stderr, " -n Don't replace, just report what would have happened\n"); fprintf(stderr, "\n"); fprintf(stderr, " The primary operation is to replace tigs in the store with ones in a set of input files.\n"); fprintf(stderr, " The input files can be either supplied directly on the command line or listed in\n"); fprintf(stderr, " a text file (-L).\n"); fprintf(stderr, "\n"); fprintf(stderr, " A new store is created if one doesn't exist, otherwise, whatever tigs are there are\n"); fprintf(stderr, " replaced with those in the -R file. If version 'v' doesn't exist, it is created.\n"); fprintf(stderr, "\n"); fprintf(stderr, " Even if -n is supplied, a new store is created if one doesn't exist.\n"); fprintf(stderr, "\n"); fprintf(stderr, " To add a new tig, give it a tig id of -1. New tigs must be added to the latest version.\n"); fprintf(stderr, " To delete a tig, remove all children, and set the number of them to zero.\n"); fprintf(stderr, "\n"); if (gkpName == NULL) fprintf(stderr, "ERROR: no gatekeeper store (-G) supplied.\n"); if (tigName == NULL) fprintf(stderr, "ERROR: no tig store (-T) supplied.\n"); if (tigInputs.size() == 0) fprintf(stderr, "ERROR: no input tigs (-R) supplied.\n"); exit(1); } // If the store doesn't exist, create one, and make a bunch of versions if (AS_UTL_fileExists(tigName, true, false) == false) { fprintf(stderr, "Creating tig store '%s' version %d\n", tigName, tigVers); tgStore *tigStore = new tgStore(tigName); for (int32 vv=1; vvnextVersion(); delete tigStore; } gkStore *gkpStore = gkStore::gkStore_open(gkpName); tgStore *tigStore = new tgStore(tigName, tigVers, tigType); tgTig *tig = new tgTig; for (uint32 ff=0; ffloadFromStreamOrLayout(TI) == true) { // Handle insertion. if (tig->numberOfChildren() > 0) { //fprintf(stderr, "INSERTING tig %d\n", tig->tigID()); tigStore->insertTig(tig, false); continue; } // Deleted already? if (tigStore->isDeleted(tig->tigID()) == true) { //fprintf(stderr, "DELETING tig %d -- ALREADY DELETED\n", tig->tigID()); continue; } // Really delete it then. //fprintf(stderr, "DELETING tig %d\n", tig->tigID()); tigStore->deleteTig(tig->tigID()); } fclose(TI); fprintf(stderr, "Reading layouts from '%s' completed.\n", tigInputs[ff]); } delete tig; delete tigStore; gkpStore->gkStore_close(); exit(0); } canu-1.3/src/stores/tgStoreLoad.mk000066400000000000000000000007271272335422300172020ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := tgStoreLoad SOURCES := tgStoreLoad.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/tgTig.C000066400000000000000000000454131272335422300156050ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2014-DEC-22 to 2015-AUG-11 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-30 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2015-NOV-25 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "tgTig.H" #include "AS_UTL_fileIO.H" #include "AS_UTL_fasta.C" #include "splitToWords.H" tgPosition::tgPosition() { _objID = UINT32_MAX; _isRead = true; // Bogus values. _isUnitig = true; _isContig = true; _isReverse = false; _spare = 0; _anchor = UINT32_MAX; _ahang = INT32_MAX; _bhang = INT32_MAX; _askip = 0; _bskip = 0; _min = INT32_MIN; _max = INT32_MAX; _deltaOffset = UINT32_MAX; _deltaLen = 0; } tgTigRecord::tgTigRecord() { _tigID = UINT32_MAX; _coverageStat = 0.0; _microhetProb = 0.0; _class = tgTig_noclass; _suggestRepeat = false; _suggestCircular = false; _spare = 0; _layoutLen = 0; _gappedLen = 0; _childrenLen = 0; _childDeltasLen = 0; } tgTig::tgTig() { _tigID = UINT32_MAX; _coverageStat = 0; _microhetProb = 0; _utgcns_verboseLevel = 0; _class = tgTig_noclass; _suggestRepeat = 0; _suggestCircular = 0; _spare = 0; _layoutLen = 0; _gappedBases = NULL; _gappedQuals = NULL; _gappedLen = 0; _gappedMax = 0; _ungappedBases = NULL; _ungappedQuals = NULL; _ungappedLen = 0; _ungappedMax = 0; _gappedToUngapped = NULL; _children = NULL; _childrenLen = 0; _childrenMax = 0; _childDeltas = NULL; _childDeltasLen = 0; _childDeltasMax = 0; } tgTig::~tgTig() { delete [] _gappedBases; delete [] _gappedQuals; delete [] _ungappedBases; delete [] _ungappedQuals; delete [] _gappedToUngapped; delete [] _children; delete [] _childDeltas; } // Copy data from an in-core tgTig to an on-disk tgTigRecord. tgTigRecord & tgTigRecord::operator=(tgTig & tg) { _tigID = tg._tigID; _coverageStat = tg._coverageStat; _microhetProb = tg._microhetProb; _class = tg._class; _suggestRepeat = tg._suggestRepeat; _suggestCircular = tg._suggestCircular; _spare = tg._spare; _layoutLen = tg._layoutLen; _gappedLen = tg._gappedLen; _childrenLen = tg._childrenLen; _childDeltasLen = tg._childDeltasLen; return(*this); } // Copy data from an on-disk tgTigRecord to an in-core tgTig. tgTig & tgTig::operator=(tgTigRecord & tr) { _tigID = tr._tigID; _coverageStat = tr._coverageStat; _microhetProb = tr._microhetProb; _class = tr._class; _suggestRepeat = tr._suggestRepeat; _suggestCircular = tr._suggestCircular; _spare = tr._spare; _layoutLen = tr._layoutLen; _gappedLen = tr._gappedLen; _childrenLen = tr._childrenLen; _childDeltasLen = tr._childDeltasLen; return(*this); } // Deep copy the tig. tgTig & tgTig::operator=(tgTig & tg) { _tigID = tg._tigID; _coverageStat = tg._coverageStat; _microhetProb = tg._microhetProb; _class = tg._class; _suggestRepeat = tg._suggestRepeat; _suggestCircular = tg._suggestCircular; _spare = tg._spare; _layoutLen = tg._layoutLen; _gappedLen = tg._gappedLen; duplicateArray(_gappedBases, _gappedLen, _gappedMax, tg._gappedBases, tg._gappedLen, tg._gappedMax); duplicateArray(_gappedQuals, _gappedLen, _gappedMax, tg._gappedQuals, tg._gappedLen, tg._gappedMax, true); if (_gappedLen > 0) { assert(_gappedMax > _gappedLen); _gappedBases[_gappedLen] = 0; _gappedQuals[_gappedLen] = 0; } _ungappedLen = tg._ungappedLen; duplicateArray(_ungappedBases, _ungappedLen, _ungappedMax, tg._ungappedBases, tg._ungappedLen, tg._ungappedMax); duplicateArray(_ungappedQuals, _ungappedLen, _ungappedMax, tg._ungappedQuals, tg._ungappedLen, tg._ungappedMax, true); if (_ungappedLen > 0) { assert(_ungappedMax > _ungappedLen); _ungappedBases[_ungappedLen] = 0; _ungappedQuals[_ungappedLen] = 0; } duplicateArray(_gappedToUngapped, _gappedLen, _gappedMax, tg._gappedToUngapped, tg._gappedLen, tg._gappedMax, true); _childrenLen = tg._childrenLen; duplicateArray(_children, _childrenLen, _childrenMax, tg._children, tg._childrenLen, tg._childrenMax); _childDeltasLen = tg._childDeltasLen; duplicateArray(_childDeltas, _childDeltasLen, _childDeltasMax, tg._childDeltas, tg._childDeltasLen, tg._childDeltasMax); return(*this); } void tgTig::buildUngapped(void) { if (_ungappedLen > 0) // Already computed. Return what is here. return; if (_gappedLen == 0) { // No gapped sequence to convert to ungapped. fprintf(stderr, "tgTig::buildUngapped()-- WARNING: tried to build ungapped sequence for tigID %u before consensus exists.\n", tigID()); return; } // Allocate more space, if needed. We'll need no more than gappedMax. We need to stash away the // max size so we can call two resizeArray() functions. uint64 ugMax = _ungappedMax; resizeArrayPair(_ungappedBases, _ungappedQuals, 0, _ungappedMax, _gappedMax, resizeArray_doNothing); resizeArray(_gappedToUngapped, 0, ugMax, _gappedMax, resizeArray_doNothing); // gappedLen doesn't include the terminating null, but gappedMax does. // See abMultiAlign.C, among other places. if (_gappedLen >= _gappedMax) fprintf(stderr, "ERROR: gappedLen = %u >= gappedMax = %u\n", _gappedLen+1, _gappedMax); assert(_gappedLen < _gappedMax); // Copy all but the gaps. _ungappedLen = 0; for (uint32 gp=0; gp<_gappedLen; gp++) { _gappedToUngapped[gp] = _ungappedLen; if (_gappedBases[gp] == '-') continue; _ungappedBases[_ungappedLen] = _gappedBases[gp]; _ungappedQuals[_ungappedLen] = _gappedQuals[gp]; _ungappedLen++; } assert(_ungappedLen < _ungappedMax); // Terminate it. Lots of work just for printf...and getting rid of gaps. _gappedToUngapped[_gappedLen] = _ungappedLen; _ungappedBases[_ungappedLen] = 0; _ungappedQuals[_ungappedLen] = 0; } // Clears the data but doesn't release memory. The only way to do that is to delete it. void tgTig::clear(void) { _tigID = UINT32_MAX; _coverageStat = 0; _microhetProb = 0; _class = tgTig_noclass; _suggestRepeat = 0; _suggestCircular = 0; _spare = 0; _layoutLen = 0; _gappedLen = 0; _ungappedLen = 0; _childrenLen = 0; _childDeltasLen = 0; } bool tgTig::loadFromStreamOrLayout(FILE *F) { // Decide if the file contains an ASCII layout or a binary stream. It's probably rather fragile, // testing if the first byte is 't' (from 'tig') or 'T' (from 'TIGR'). char ch = getc(F); ungetc(ch, F); if (ch == 't') return(loadLayout(F)); else if (ch == 'T') return(loadFromStream(F)); else return(false); } void tgTig::saveToStream(FILE *F) { tgTigRecord tr = *this; char tag[4] = {'T', 'I', 'G', 'R', }; // That's tigRecord, not TIGR AS_UTL_safeWrite(F, tag, "tgTig::saveToStream::tigr", sizeof(char), 4); AS_UTL_safeWrite(F, &tr, "tgTig::saveToStream::tr", sizeof(tgTigRecord), 1); // We could save the null byte too, but don't. It's explicitly added during the load. if (_gappedLen > 0) { AS_UTL_safeWrite(F, _gappedBases, "tgTig::saveToStream::gappedBases", sizeof(char), _gappedLen); AS_UTL_safeWrite(F, _gappedQuals, "tgTig::saveToStream::gappedQuals", sizeof(char), _gappedLen); } if (_childrenLen > 0) AS_UTL_safeWrite(F, _children, "tgTig::saveToStream::children", sizeof(tgPosition), _childrenLen); if (_childDeltasLen > 0) AS_UTL_safeWrite(F, _childDeltas, "tgTig::saveToStream::childDeltas", sizeof(int32), _childDeltasLen); } bool tgTig::loadFromStream(FILE *F) { char tag[4]; clear(); // Read the tgTigRecord from disk and copy it into our tgTig. tgTigRecord tr; if (4 != AS_UTL_safeRead(F, tag, "tgTig::saveToStream::tigr", sizeof(char), 4)) { return(false); } if ((tag[0] != 'T') || (tag[1] != 'I') || (tag[2] != 'G') || (tag[3] != 'R')) { return(false); } if (0 == AS_UTL_safeRead(F, &tr, "tgTig::loadFromStream::tr", sizeof(tgTigRecord), 1)) { // Nothing loaded, end of file. return(false); } *this = tr; // Allocate space for bases/quals and load them. Be sure to terminate them, too. resizeArrayPair(_gappedBases, _gappedQuals, 0, _gappedMax, _gappedLen + 1, resizeArray_doNothing); if (_gappedLen > 0) { AS_UTL_safeRead(F, _gappedBases, "tgTig::loadFromStream::gappedBases", sizeof(char), _gappedLen); AS_UTL_safeRead(F, _gappedQuals, "tgTig::loadFromStream::gappedQuals", sizeof(char), _gappedLen); _gappedBases[_gappedLen] = 0; _gappedQuals[_gappedLen] = 0; } // Allocate space for reads and alignments, and load them. resizeArray(_children, 0, _childrenMax, _childrenLen, resizeArray_doNothing); resizeArray(_childDeltas, 0, _childDeltasMax, _childDeltasLen, resizeArray_doNothing); if (_childrenLen > 0) AS_UTL_safeRead(F, _children, "tgTig::savetoStream::children", sizeof(tgPosition), _childrenLen); if (_childDeltasLen > 0) AS_UTL_safeRead(F, _childDeltas, "tgTig::loadFromStream::childDeltas", sizeof(int32), _childDeltasLen); // Return success. return(true); }; void tgTig::dumpLayout(FILE *F) { char deltaString[128] = {0}; char trimString[128] = {0}; if (_gappedLen > 0) assert(_gappedLen == _layoutLen); fprintf(F, "tig "F_U32"\n", _tigID); fprintf(F, "len %d\n", _layoutLen); // Adjust QV's to Sanger encoding for (uint32 ii=0; ii<_gappedLen; ii++) _gappedQuals[ii] += '!'; // Dump the sequence and quality if (_gappedLen == 0) { fputs("cns\n", F); fputs("qlt\n", F); } else { fputs("cns ", F); fputs(_gappedBases, F); fputs("\n", F); // strings are null terminated now, but expected to be long. fputs("qlt ", F); fputs(_gappedQuals, F); fputs("\n", F); } // Adjust QV's back to no encoding for (uint32 ii=0; ii<_gappedLen; ii++) _gappedQuals[ii] -= '!'; // Properties. fprintf(F, "coverageStat %f\n", _coverageStat); fprintf(F, "microhetProb %f\n", _microhetProb); fprintf(F, "class %s\n", toString(_class)); fprintf(F, "suggestRepeat %c\n", _suggestRepeat ? 'T' : 'F'); fprintf(F, "suggestCircular %c\n", _suggestCircular ? 'T' : 'F'); fprintf(F, "numChildren "F_U32"\n", _childrenLen); // And the reads. for (uint32 i=0; i<_childrenLen; i++) { tgPosition *imp = _children + i; trimString[0] = 0; deltaString[0] = 0; if (imp->_askip + imp->_bskip > 0) sprintf(trimString, " trim %6u %6u", imp->_askip, imp->_bskip); if (imp->_deltaLen > 0) sprintf(deltaString, " delta %5u at %u", imp->_deltaLen, imp->_deltaOffset); if (imp->_isRead) fprintf(F, "read %9"F_U32P" anchor %9"F_U32P" hang %6"F_S32P" %6"F_S32P" position %6"F_U32P" %6"F_U32P"%s%s\n", imp->ident(), imp->anchor(), imp->aHang(), imp->bHang(), imp->bgn(), imp->end(), trimString, deltaString); if (imp->_isUnitig) fprintf(F, "unitig %9"F_U32P" anchor %9"F_U32P" hang %6"F_S32P" %6"F_S32P" position %6"F_U32P" %6"F_U32P"%s%s\n", imp->ident(), imp->anchor(), imp->aHang(), imp->bHang(), imp->bgn(), imp->end(), trimString, deltaString); if (imp->_isContig) fprintf(F, "contig %9"F_U32P" anchor %9"F_U32P" hang %6"F_S32P" %6"F_S32P" position %6"F_U32P" %6"F_U32P"%s%s\n", imp->ident(), imp->anchor(), imp->aHang(), imp->bHang(), imp->bgn(), imp->end(), trimString, deltaString); } fprintf(F, "tigend\n"); } bool tgTig::loadLayout(FILE *F) { uint64 LINEnum = 0; uint32 LINElen = 0; uint32 LINEmax = 1 * 1024 * 1024; char *LINE = new char [LINEmax]; uint32 nChildren = 0; clear(); fgets(LINE, LINEmax, F); LINEnum++; if (feof(F)) return(false); while (!feof(F)) { splitToWords W(LINE); if ((W.numWords() == 0) || (W[0][0] == '#') || (W[0][0] == '!')) { // Comment, ignore. } else if (strcmp(W[0], "tig") == 0) { _tigID = strtouint32(W[1]); } else if (strcmp(W[0], "len") == 0) { _layoutLen = strtouint32(W[1]); resizeArray(LINE, LINElen, LINEmax, _layoutLen + 1, resizeArray_doNothing); } else if (((strcmp(W[0], "cns") == 0) || (strcmp(W[0], "qlt") == 0)) && (W.numWords() == 1)) { _gappedLen = 0; } else if (((strcmp(W[0], "cns") == 0) || (strcmp(W[0], "qlt") == 0)) && (W.numWords() == 2)) { _gappedLen = strlen(W[1]); _layoutLen = _gappedLen; // Must be enforced, probably should be an explicit error. resizeArrayPair(_gappedBases, _gappedQuals, 0, _gappedMax, _gappedLen+1, resizeArray_doNothing); if (W[0][0] == 'c') memcpy(_gappedBases, W[1], sizeof(char) * (_gappedLen + 1)); // W[1] is null terminated, and we just copy it in else memcpy(_gappedQuals, W[1], sizeof(char) * (_gappedLen + 1)); } else if (strcmp(W[0], "coverageStat") == 0) { _coverageStat = strtodouble(W[1]); } else if (strcmp(W[0], "microhetProb") == 0) { _microhetProb = strtodouble(W[1]); } else if (strcmp(W[0], "class") == 0) { if (strcmp(W[1], "unassembled") == 0) _class = tgTig_unassembled; else if (strcmp(W[1], "bubble") == 0) _class = tgTig_bubble; else if (strcmp(W[1], "contig") == 0) _class = tgTig_contig; else fprintf(stderr, "tgTig::loadLayout()-- '%s' line "F_U64" invalid: '%s'\n", W[0], LINEnum, LINE), exit(1); } else if (strcmp(W[0], "suggestRepeat") == 0) { _suggestRepeat = strtouint32(W[1]); } else if (strcmp(W[0], "suggestCircular") == 0) { _suggestCircular = strtouint32(W[1]); } else if (strcmp(W[0], "numChildren") == 0) { //_numChildren = strtouint32(W[1]); //resizeArray(_children, 0, _childrenMax, _childrenLen, resizeArray_doNothing); } else if ((strcmp(W[0], "read") == 0) || (strcmp(W[0], "unitig") == 0) || (strcmp(W[0], "contig") == 0)) { if (W.numWords() < 10) fprintf(stderr, "tgTig::loadLayout()-- '%s' line "F_U64" invalid: '%s'\n", W[0], LINEnum, LINE), exit(1); if (nChildren >= _childrenLen) { resizeArray(_children, _childrenLen, _childrenMax, _childrenLen + 1, resizeArray_copyData); _childrenLen++; } _children[nChildren]._objID = strtouint32(W[1]); _children[nChildren]._isRead = (strcmp(W[0], "read") == 0); _children[nChildren]._isUnitig = (strcmp(W[0], "unitig") == 0); _children[nChildren]._isContig = (strcmp(W[0], "contig") == 0); _children[nChildren]._isReverse = false; _children[nChildren]._spare = 0; _children[nChildren]._anchor = strtouint32(W[3]); _children[nChildren]._ahang = strtouint32(W[5]); _children[nChildren]._bhang = strtouint32(W[6]); _children[nChildren]._askip = 0; _children[nChildren]._bskip = 0; _children[nChildren]._min = strtouint32(W[8]); _children[nChildren]._max = strtouint32(W[9]); _children[nChildren]._deltaOffset = 0; _children[nChildren]._deltaLen = 0; if (_children[nChildren]._max < _children[nChildren]._min) { _children[nChildren]._min = strtouint32(W[9]); _children[nChildren]._max = strtouint32(W[8]); _children[nChildren]._isReverse = true; } for (uint32 pos=10; (pos < W.numWords()); pos++) { if (strcmp(W[pos], "delta") == 0) { _children[nChildren]._deltaLen = strtouint32(W[++pos]); pos++; // "at" _children[nChildren]._deltaOffset = strtouint32(W[++pos]); } if (strcmp(W[pos], "trim") == 0) { _children[nChildren]._askip = strtouint32(W[++pos]); _children[nChildren]._bskip = strtouint32(W[++pos]); } pos++; } nChildren++; } else if (strcmp(W[0], "tigend") == 0) { // All done, get out of the reading loop. break; } else { // LINE is probably munged by splitToWords. fprintf(stderr, "tgTig::loadLayout()-- unknown line '%s'\n", LINE); } fgets(LINE, LINEmax, F); LINEnum++; } delete [] LINE; return(true); } void tgTig::dumpFASTA(FILE *F, bool useGapped) { AS_UTL_writeFastA(F, bases(useGapped), length(useGapped), 100, ">tig%08u len="F_U32" reads="F_U32" covStat=%.2f gappedBases=%s class=%s suggestRepeat=%s suggestCircular=%s\n", tigID(), length(useGapped), numberOfChildren(), _coverageStat, (useGapped) ? "yes" : "no", toString(_class), _suggestRepeat ? "yes" : "no", _suggestCircular ? "yes" : "no"); } void tgTig::dumpFASTQ(FILE *F, bool useGapped) { AS_UTL_writeFastQ(F, bases(useGapped), length(useGapped), quals(useGapped), length(useGapped), "@tig%08u len="F_U32" reads="F_U32" covStat=%.2f gappedBases=%s class=%s suggestRepeat=%s suggestCircular=%s\n", tigID(), length(useGapped), numberOfChildren(), _coverageStat, (useGapped) ? "yes" : "no", toString(_class), _suggestRepeat ? "yes" : "no", _suggestCircular ? "yes" : "no"); } canu-1.3/src/stores/tgTig.H000066400000000000000000000307241272335422300156110ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2014-DEC-22 to 2015-AUG-14 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-30 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef TG_TIG_H #define TG_TIG_H #include "AS_global.H" #include "gkStore.H" typedef enum { tgTig_noclass = 0x00, // Could use a tgTig_read for corrections tgTig_unassembled = 0x01, tgTig_bubble = 0x02, tgTig_contig = 0x03 } tgTig_class; static const char * toString(tgTig_class c) { switch (c) { case tgTig_noclass: return("unsetc"); break; case tgTig_unassembled: return("unassm"); break; case tgTig_bubble: return("bubble"); break; case tgTig_contig: return("contig"); break; } return("undefined-class"); } static tgTig_class toClass(char const *c) { if (strcmp(c, "unsetc") == 0) return(tgTig_noclass); if (strcmp(c, "unassm") == 0) return(tgTig_unassembled); if (strcmp(c, "bubble") == 0) return(tgTig_bubble); if (strcmp(c, "contig") == 0) return(tgTig_contig); fprintf(stderr, "WARNING: tiTig_class toClass('%s') is not a valid class.\n", c); return(tgTig_noclass); } // Info about the placement of an object in a tig. For unitigs, this // will be just reads. For contigs, this could be unitigs and reads. // // These coordinates are ALWAYS gapped coordinates. // class tgPosition { public: tgPosition(); // Accessors uint32 ident(void) const { return(_objID); }; bool isRead(void) const { return(_isRead == true); }; bool isUnitig(void) const { return(_isUnitig == true); }; bool isContig(void) const { return(_isContig == true); }; uint32 anchor(void) { return(_anchor); }; int32 aHang(void) { return(_ahang); }; int32 bHang(void) { return(_bhang); }; bool isForward(void) const { return(_isReverse == false); }; bool isReverse(void) const { return(_isReverse == true); }; // Position in the parent, both oriented (bgn/end) and unoriented (min/max). int32 bgn(void) const { return((_isReverse == false) ? _min : _max); }; int32 end(void) const { return((_isReverse == false) ? _max : _min); }; int32 min(void) const { return(_min); }; int32 max(void) const { return(_max); }; // Amount of this object to ignore; e.g., trim from the ends. int32 askip(void) const { return(_askip); }; int32 bskip(void) const { return(_bskip); }; // Delta values for the alignment to the parent. uint32 deltaOffset(void) { return(_deltaOffset); }; uint32 deltaLength(void) { return(_deltaLen); }; // Set just the anchor and hangs, leave positions alone. void setAnchor(uint32 anchor, int32 ahang, int32 bhang) { _anchor = anchor; _ahang = ahang; _bhang = bhang; }; // Set everything. This is to be used by unitigger. void set(uint32 id, uint32 anchor, int32 ahang, int32 bhang, int32 bgn, int32 end) { _objID = id; _isRead = true; _isUnitig = false; _isContig = false; _anchor = anchor; _ahang = ahang; _bhang = bhang; _askip = 0; _bskip = 0; if (bgn < end) { _min = bgn; _max = end; _isReverse = false; } else { _min = end; _max = bgn; _isReverse = true; } _deltaOffset = 0; _deltaLen = 0; }; // Set the coords, ignoring orientation. void setMinMax(int32 min, int32 max) { _min = min; _max = max; }; // Operators bool operator<(tgPosition const &that) const { int32 al = min(); int32 bl = that.min(); if (al != bl) return(al < bl); if (that._anchor == _objID) // I'm his anchor, I must be before it. return(true); if (_anchor == that._objID) // My anchor is the other tig; I must return(false); // be after it. int32 ah = max(); int32 bh = that.max(); return(ah > bh); } private: public: uint32 _objID; // ID of this object uint32 _isRead : 1; // A full length read alignment uint32 _isUnitig : 1; // uint32 _isContig : 1; // uint32 _isReverse : 1; // Child is oriented forward relative to parent, used during consensus. uint32 _spare : 28; uint32 _anchor; // ID of the like-type object we align to int32 _ahang; // Placement relative to anchor object int32 _bhang; // int32 _askip; // Amount of sequence to not align on each end int32 _bskip; // // Must be signed, utgcns can push reads negative. //int32 _bgn; // Coords in the parent object we are part of //int32 _end; // (for a read, this will be the position in the unitig) // Must be signed, utgcns can push reads negative. int32 _min; int32 _max; uint32 _deltaOffset; // Our delta alignment to the parent object consensus uint32 _deltaLen; }; class tgTig; // Early declaration, for use in tgTigRecord operator= // On-disk tig, same as tgTig without the pointers class tgTigRecord { public: tgTigRecord(); tgTigRecord(tgTig &tg) { *this = tg; }; // to support tgTigRecord tr = tgtig tgTigRecord &operator=(tgTig & tg); private: public: uint32 _tigID; double _coverageStat; double _microhetProb; // placeholder for some future multialignment based uniqueness probability tgTig_class _class : 2; // Output classification: unassembled, bubble, contig uint32 _suggestRepeat : 1; // Bogart made this from detected repeat. uint32 _suggestCircular : 1; // Bogart found overlaps making a circle. uint32 _spare : 32 - 2 - 2; uint32 _layoutLen; // Max coord uint32 _gappedLen; // Gapped consensus uint32 _childrenLen; uint32 _childDeltasLen; }; // Former MultiAlignT // In core data class tgTig { public: tgTig(); // All data unallocated, lengths set to zero ~tgTig(); // Releases memory // Accessors uint32 tigID(void) { return(_tigID); }; char const *coordinateType(bool useGapped=true) { return( (consensusExists() == false) ? ("layout") : ((useGapped == true) ? "gapped" : "ungapped") ); }; uint32 length(bool useGapped=true) { return( (consensusExists() == false) ? (layoutLength()) : ((useGapped == true) ? gappedLength() : ungappedLength()) ); }; bool consensusExists(void) { return(_gappedLen > 0); }; char *bases(bool useGapped=true) { return( (useGapped == true) ? gappedBases() : ungappedBases() ); }; char *quals(bool useGapped=true) { return( (useGapped == true) ? gappedQuals() : ungappedQuals() ); }; private: uint32 layoutLength(void) { return(_layoutLen); }; uint32 gappedLength(void) { return(_gappedLen); }; char *gappedBases(void) { return(_gappedBases); }; char *gappedQuals(void) { return(_gappedQuals); }; void buildUngapped(void); uint32 ungappedLength(void) { buildUngapped(); return(_ungappedLen); }; char *ungappedBases(void) { buildUngapped(); return(_ungappedBases); }; char *ungappedQuals(void) { buildUngapped(); return(_ungappedQuals); }; public: // This function needs to be hidden. Coordinates in reads need to be transparent. uint32 mapGappedToUngapped(uint32 p) { if (consensusExists() == false) return(p); buildUngapped(); return(_gappedToUngapped[p]); }; uint32 numberOfChildren(void) { return(_childrenLen); }; tgPosition *getChild(uint32 c) { assert(c < _childrenLen); return(_children + c); }; tgPosition *addChild(void) { return(_children + _childrenLen++); }; // Operators void clear(void); // Clears data but doesn't release memory. tgTig &operator=(tgTigRecord & tr); tgTig &operator=(tgTig &tg); bool loadFromStreamOrLayout(FILE *F); void saveToStream(FILE *F); bool loadFromStream(FILE *F); void dumpLayout(FILE *F); bool loadLayout(FILE *F); void dumpFASTA(FILE *F, bool useGapped); void dumpFASTQ(FILE *F, bool useGapped); // There are two multiAlign displays; this one, and one in abMultiAlign. void display(FILE *F, gkStore *gkp, uint32 displayWidth = 100, // Width of display uint32 displaySpacing = 3, // Space between reads on the same line bool withQV = false, bool withDots = false); private: public: uint32 _tigID; // ID in the store, or UINT32_MAX if not set double coverageStat(void) { return(_coverageStat); }; //double microhetProb(void) { return(_microhetProb); }; double _coverageStat; double _microhetProb; // Placeholder. // A variety of flags to suggest what type of unitig this is tgTig_class _class : 2; // Output classification: unassembled, bubble, contig uint32 _suggestRepeat : 1; // Bogart made this from detected repeat. uint32 _suggestCircular : 1; // Bogart found overlaps making a circle. uint32 _spare : 32 - 2 - 2; uint32 _layoutLen; // The max coord in the layout. Same as gappedLen if it exists. char *_gappedBases; // Gapped consensus - used by the multialignment. NUL terminated. char *_gappedQuals; uint32 _gappedLen; // Doesn't include the NUL. uint32 _gappedMax; // Does include the NUL. char *_ungappedBases; // Ungapped consensus - not used by the assember, only output. char *_ungappedQuals; uint32 _ungappedLen; uint32 _ungappedMax; uint32 *_gappedToUngapped; // Map a gapped position to an ungapped posision, only output. tgPosition *_children; // positions of objects that make up this tig uint32 _childrenLen; uint32 _childrenMax; int32 *_childDeltas; // deltas for all objects in the _children list uint32 _childDeltasLen; uint32 _childDeltasMax; // Flags for computing consensus/multialignments uint32 _utgcns_verboseLevel; }; #endif canu-1.3/src/stores/tgTigDisplay.C000066400000000000000000000040271272335422300171270ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-JAN-26 to 2015-JUL-01 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-07 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" int main(int argc, char **argv) { tgTig tig; char *gkpName = NULL; char *tigFileName = NULL; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-t") == 0) { tigFileName = argv[++arg]; } else { err++; } arg++; } if (gkpName == NULL) err++; if (tigFileName == NULL) err++; if (err) { fprintf(stderr, "usage: %s -G gkpStore -t tigFile\n", argv[0]); exit(1); } gkStore *gkpStore = gkStore::gkStore_open(gkpName); FILE *F = fopen(tigFileName, "r"); tig.loadFromStream(F); fclose(F); uint32 displayWidth = 250; uint32 displaySpacing = 10; bool withQV = false; bool withDots = true; tig.display(stdout, gkpStore, displayWidth, displaySpacing, withQV, withDots); exit(0); } canu-1.3/src/stores/tgTigDisplay.mk000066400000000000000000000007311272335422300173520ustar00rootroot00000000000000 # If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := tgTigDisplay SOURCES := tgTigDisplay.C SRC_INCDIRS := .. ../AS_UTL TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES := canu-1.3/src/stores/tgTigMultiAlignDisplay.C000066400000000000000000000277321272335422300211250ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignPrint.C * src/AS_CNS/MultiAlignPrint.c * * Modifications by: * * Brian P. Walenz from 2007-SEP-05 to 2013-AUG-01 * are Copyright 2007-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-21 to 2015-JUN-03 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-03 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" #include "AS_UTL_reverseComplement.H" #include class LaneNode { public: LaneNode() { read = NULL; readLen = 0; bases = NULL; quals = NULL; next = NULL; }; ~LaneNode() { delete [] bases; delete [] quals; }; tgPosition *read; int32 readLen; char *bases; // Allocated bases char *quals; // Allocated quals int32 *delta; // Pointer to tgTig _childDeltas LaneNode *next; }; class Lane { public: Lane() { first = NULL; last = NULL; lastcol = 0; }; ~Lane() { LaneNode *node = first; LaneNode *next = NULL; while (node) { next = node->next; delete node; node = next; } }; // Add a node to this lane, if possible. bool addNode(LaneNode *node, uint32 displaySpacing) { int32 leftpos = node->read->min(); if ((lastcol > 0) && (leftpos < lastcol + displaySpacing)) return(false); assert(node->next == NULL); if (first == NULL) { first = node; last = node; } else { last->next = node; last = node; } lastcol = leftpos + node->read->deltaLength() + node->readLen; return(true); }; LaneNode *first; LaneNode *last; int32 lastcol; }; void tgTig::display(FILE *F, gkStore *gkp, uint32 displayWidth, uint32 displaySpacing, bool withQV, bool withDots) { if (gappedLength() == 0) { fprintf(F, "No MultiAlignment to print for tig %d -- no consensus sequence present.\n", tigID()); return; } fprintf(stderr, "tgTig::display()-- display tig %d with %d children\n", tigID(), _childrenLen); fprintf(stderr, "tgTig::display()-- width %u spacing %u\n", displayWidth, displaySpacing); // // Convert the children to a list of lines to print // Lane *lane = NULL; Lane *lanes = new Lane [_childrenLen]; int32 lanesLen = 0; int32 lanesPos = 0; // Sort the fragments by leftmost position within tig std::sort(_children, _children + _childrenLen); // Load into lanes. gkReadData *readData = new gkReadData; for (int32 i=0; i<_childrenLen; i++) { gkRead *read = gkp->gkStore_getRead(_children[i].ident()); // Too many reads in this code. gkp->gkStore_loadReadData(read, readData); LaneNode *node = new LaneNode(); node->read = _children + i; node->readLen = read->gkRead_sequenceLength(); node->bases = new char [node->readLen + 1]; node->quals = new char [node->readLen + 1]; node->delta = _childDeltas + node->read->deltaOffset(); memcpy(node->bases, readData->gkReadData_getSequence(), sizeof(char) * node->readLen); memcpy(node->quals, readData->gkReadData_getQualities(), sizeof(char) * node->readLen); node->bases[node->readLen] = 0; node->quals[node->readLen] = 0; for (uint32 ii=0; iireadLen; ii++) // Adjust QVs for display node->quals[ii] += '!'; if (node->read->isReverse()) reverseComplement(node->bases, node->quals, node->readLen); //fprintf(stderr, "NODE READ %u at %d %d tigLength %d with %u deltas at offset %u\n", // i, node->read->bgn(), node->read->end(), gappedLength(), node->read->deltaLength(), node->read->deltaOffset()); // Try to add this new node to the lanes. The last iteration will always succeed, adding the // node to a fresh empty lane. for (lanesPos=0; lanesPos <= lanesLen; lanesPos++) if (lanes[lanesPos].addNode(node, displaySpacing)) break; assert(lanesPos <= lanesLen); // If it is added to the last one, increment our cnsLen. if (lanesPos == lanesLen) lanesLen++; } delete readData; // Allocate space. char **multia = new char * [2 * lanesLen]; for (int32 i=0; i<2*lanesLen; i++) { multia[i] = new char [gappedLength() + displayWidth + 1]; memset(multia[i], ' ', gappedLength() + displayWidth); multia[i][gappedLength() + displayWidth] = 0; } int32 **idarray = new int32 * [lanesLen]; int32 **oriarray = new int32 * [lanesLen]; // Not sure where the +1 comes from. The original was using GetNumchars(ma->consensus) which (I // think) included the terminating NUL, while gappedLength() does not. for (int32 i=0; inext) { int32 firstcol = (node->read->bgn() < node->read->end()) ? node->read->bgn() : node->read->end(); int32 lastcol = (node->read->bgn() < node->read->end()) ? node->read->end() : node->read->bgn(); int32 orient = (node->read->bgn() < node->read->end()) ? 1 : -1; if (lastcol > gappedLength()) fprintf(stderr, "lastcol too big: %d vs %d\n", lastcol, gappedLength()); assert(firstcol <= gappedLength()); assert(lastcol <= gappedLength()); // Set ID and orientation //fprintf(stderr, "READ %d minmax %d %d bgnend %d %d orient %d\n", // node->read->ident(), node->read->min(), node->read->max(), node->read->bgn(), node->read->end(), orient); for (int32 col=firstcol; colread->ident(); oriarray[i][col] = orient; } // Set bases int32 col = firstcol; int32 cols = 0; for (int32 j=0; jread->deltaLength(); j++) { int32 seglen = node->delta[j] - ((j > 0) ? node->delta[j-1] : 0); if (cols + seglen >= node->readLen) fprintf(stderr, "ERROR: Clear ranges not correct.\n"); assert(cols + seglen < node->readLen); //fprintf(stderr, "copy to col=%d from cols=%d seglen=%d -- max %d -- delta %d\n", // col, cols, seglen, gappedLength() + displayWidth, node->delta[j]); memcpy(srow + col, node->bases + cols, seglen); memcpy(qrow + col, node->quals + cols, seglen); col += seglen; srow[col] = '-'; qrow[col] = '-'; col++; cols += seglen; } memcpy(srow + col, node->bases + cols, node->readLen - cols); memcpy(qrow + col, node->quals + cols, node->readLen - cols); } } // Cleanup delete [] lanes; // // // fprintf(F, "<<< begin Contig %d >>>", tigID()); char *gruler = new char [displayWidth + 200]; char *uruler = new char [displayWidth + 200]; int32 ungapped = 1; int32 tick = 1; // The original used 'length = strlen(consensus)' which does NOT include the terminating NUL. for (uint32 window=0; window < gappedLength(); ) { uint32 row_id = 0; uint32 orient = 0; uint32 rowlen = (window + displayWidth < gappedLength()) ? displayWidth : gappedLength() - window; fprintf(F, "\n"); fprintf(F, "\n"); fprintf(F, "<<< tig %d, gapped length: %d >>>\n", tigID(), gappedLength()); { memset(gruler, 0, displayWidth + 200); memset(uruler, 0, displayWidth + 200); for (uint32 rowind=0; rowind= 0) && (gruler[i] == ' '); i--) gruler[i] = 0; for (int32 i=displayWidth-1; (i >= 0) && (uruler[i] == ' '); i--) uruler[i] = 0; fprintf(F, "%s\n", gruler); fprintf(F, "%s\n", uruler); } { char save = _gappedBases[window + rowlen]; _gappedBases[window + rowlen] = 0; fprintf(F, "%s cns (iid) type\n", _gappedBases + window); _gappedBases[window + rowlen] = save; } { for (uint32 ii=0; ii gappedLength()) break; if (multia[2*i][window+j] == _gappedBases[window+j]) { if (withDots) { multia[2*i] [window+j] = '.'; multia[2*i+1][window+j] = ' '; } else { multia[2*i][window+j] = tolower(multia[2*i][window+j]); } } if (multia[2*i][window+j] != ' ') nonBlank++; if (idarray[i][window + j] > 0) { row_id = idarray[i][window + j]; orient = oriarray[i][window + j]; } } if (nonBlank == 0) continue; // Figure out the ID and orientation for this block { char save = multia[2*i][window + displayWidth]; multia[2*i][window + displayWidth] = 0; fprintf(F, "%s %c (%d)\n", multia[2*i] + window, (orient>0)?'>':'<', row_id); multia[2*i][window + displayWidth] = save; } if (withQV) { char save = multia[2*i+1][window + displayWidth]; multia[2*i+1][window + displayWidth] = 0; fprintf(F, "%s\n", multia[2*i+1] + window); multia[2*i+1][window + displayWidth] = save; } } window += displayWidth; } fprintf(F, "\n<<< end Contig %d >>>\n", tigID()); delete [] uruler; delete [] gruler; for (uint32 i=0; i < 2 * lanesLen; i++) delete [] multia[i]; delete [] multia; for (uint32 i=0; i < lanesLen; i++) { delete [] idarray[i]; delete [] oriarray[i]; } delete [] idarray; delete [] oriarray; //fprintf(stderr, "Return.\n"); } canu-1.3/src/stores/tgTigSizeAnalysis.C000066400000000000000000000075731272335422300201510ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignSizeAnalysis.C * * Modifications by: * * Brian P. Walenz from 2012-MAR-26 to 2013-OCT-24 * are Copyright 2012-2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz on 2014-DEC-22 * are Copyright 2014 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-30 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "tgTigSizeAnalysis.H" #include #include #include #include #include using namespace std; tgTigSizeAnalysis::tgTigSizeAnalysis(uint64 genomeSize_) { genomeSize = genomeSize_; } tgTigSizeAnalysis::~tgTigSizeAnalysis() { } void tgTigSizeAnalysis::evaluateTig(tgTig *tig, bool useGapped) { // Try to get the ungapped length. // But revert to the gapped length if that doesn't exist. This should // only occur for pre-consensus unitigs. uint32 length = tig->length(useGapped); if (tig->_suggestRepeat) lenSuggestRepeat.push_back(length); if (tig->_suggestCircular) lenSuggestCircular.push_back(length); switch (tig->_class) { case tgTig_unassembled: lenUnassembled.push_back(length); break; case tgTig_bubble: lenBubble.push_back(length); break; case tgTig_contig: lenContig.push_back(length); break; } } void tgTigSizeAnalysis::finalize(void) { sort(lenSuggestRepeat.begin(), lenSuggestRepeat.end(), greater()); sort(lenSuggestCircular.begin(), lenSuggestCircular.end(), greater()); sort(lenUnassembled.begin(), lenUnassembled.end(), greater()); sort(lenBubble.begin(), lenBubble.end(), greater()); sort(lenContig.begin(), lenContig.end(), greater()); } void tgTigSizeAnalysis::printSummary(FILE *out, char *description, vector &data) { uint64 cnt = data.size(); uint64 sum = 0; uint64 tot = 0; uint64 nnn = 10; uint64 siz = 0; // Duplicates AS_BAT_Instrumentation.C reportN50(). if (cnt == 0) return; for (uint64 i=0; i 0) siz = genomeSize; else siz = tot; for (uint64 i=0; i using namespace std; class tgTigSizeAnalysis { public: tgTigSizeAnalysis(uint64 genomeSize); ~tgTigSizeAnalysis(); void evaluateTig(tgTig *tig, bool useGapped=true); void finalize(void); void printSummary(FILE *out, char *description, vector &data); void printSummary(FILE *out); private: uint64 genomeSize; vector lenSuggestRepeat; vector lenSuggestCircular; vector lenUnassembled; vector lenBubble; vector lenContig; }; #endif // TGTIGSIZEANALYSIS canu-1.3/src/utgcns/000077500000000000000000000000001272335422300144005ustar00rootroot00000000000000canu-1.3/src/utgcns/libNDFalcon/000077500000000000000000000000001272335422300165135ustar00rootroot00000000000000canu-1.3/src/utgcns/libNDFalcon/LICENSE000066400000000000000000000035521272335422300175250ustar00rootroot00000000000000#################################################################################$$ # Copyright (c) 2011-2015, Pacific Biosciences of California, Inc. # # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted (subject to the limitations in the # disclaimer below) provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # # * Neither the name of Pacific Biosciences nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE # GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC # BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED # WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. #################################################################################$$ canu-1.3/src/utgcns/libNDFalcon/dw.C000066400000000000000000000267241272335422300172440ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2015-DEC-28 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ /* * ===================================================================================== * * Filename: DW.c * * Description: A banded version for the O(ND) greedy sequence alignment algorithm * * Version: 0.1 * Created: 07/20/2013 17:00:00 * Revision: none * Compiler: gcc * * Author: Jason Chin, * Company: * * ===================================================================================== #################################################################################$$ # Copyright (c) 2011-2014, Pacific Biosciences of California, Inc. # # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted (subject to the limitations in the # disclaimer below) provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # # * Neither the name of Pacific Biosciences nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE # GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC # BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED # WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. #################################################################################$$ */ #include #include #include #include #include "dw.H" namespace NDalignment { int compare_d_path(const void * a, const void * b) { const d_path_data2 * arg1 = (d_path_data2 *)a; const d_path_data2 * arg2 = (d_path_data2 *)b; if (arg1->d - arg2->d == 0) { return arg1->k - arg2->k; } else { return arg1->d - arg2->d; } } void d_path_sort( d_path_data2 * base, unsigned long max_idx) { qsort(base, max_idx, sizeof(d_path_data2), compare_d_path); } d_path_data2 * get_dpath_idx( seq_coor_t d, seq_coor_t k, unsigned long max_idx, d_path_data2 * base) { d_path_data2 d_tmp; d_path_data2 *rtn; d_tmp.d = d; d_tmp.k = k; rtn = (d_path_data2 *) bsearch( &d_tmp, base, max_idx, sizeof(d_path_data2), compare_d_path); //printf("dp %ld %ld %ld %ld %ld %ld %ld\n", (rtn)->d, (rtn)->k, (rtn)->x1, (rtn)->y1, (rtn)->x2, (rtn)->y2, (rtn)->pre_k); return rtn; } void print_d_path( d_path_data2 * base, unsigned long max_idx) { unsigned long idx; for (idx = 0; idx < max_idx; idx++){ printf("dp %ld %d %d %d %d %d %d %d\n",idx, (base+idx)->d, (base+idx)->k, (base+idx)->x1, (base+idx)->y1, (base+idx)->x2, (base+idx)->y2, (base+idx)->pre_k); } } bool align(const char * query_seq, seq_coor_t q_len, const char * target_seq, seq_coor_t t_len, seq_coor_t band_tolerance, bool get_aln_str, NDalignResult &align_rtn) { seq_coor_t * V; seq_coor_t * U; // array of matched bases for each "k" seq_coor_t k_offset; seq_coor_t d; seq_coor_t k, k2; seq_coor_t best_m; // the best "matches" for each d seq_coor_t min_k, new_min_k; seq_coor_t max_k, new_max_k; seq_coor_t pre_k; seq_coor_t x, y; seq_coor_t cd; seq_coor_t ck; seq_coor_t cx, cy, nx, ny; seq_coor_t max_d; seq_coor_t band_size; unsigned long d_path_idx = 0; unsigned long max_idx = 0; d_path_data2 * d_path; d_path_data2 * d_path_aux; path_point * aln_path; seq_coor_t aln_path_idx; seq_coor_t aln_pos; seq_coor_t i; bool aligned = false; //printf("debug: %ld %ld\n", q_len, t_len); //printf("%s\n", query_seq); max_d = (int) (0.3*(q_len + t_len)); band_size = band_tolerance * 2; V = (seq_coor_t *)calloc( max_d * 2 + 1, sizeof(seq_coor_t) ); U = (seq_coor_t *)calloc( max_d * 2 + 1, sizeof(seq_coor_t) ); k_offset = max_d; // We should probably use hashmap to store the backtracing information to save memory allocation time // This O(MN) block allocation scheme is convient for now but it is slower for very long sequences d_path = (d_path_data2 *)calloc( max_d * (band_size + 1 ) * 2 + 1, sizeof(d_path_data2) ); aln_path = (path_point *)calloc( q_len + t_len + 1, sizeof(path_point) ); if (get_aln_str) { align_rtn._tgt_aln_str = (char *)calloc( q_len + t_len + 1, sizeof(char)); align_rtn._qry_aln_str = (char *)calloc( q_len + t_len + 1, sizeof(char)); } else { align_rtn._tgt_aln_str = NULL; align_rtn._qry_aln_str = NULL; } align_rtn._size = 0; align_rtn._qry_bgn = 0; align_rtn._qry_end = 0; align_rtn._tgt_bgn = 0; align_rtn._tgt_end = 0; //printf("max_d: %lu, band_size: %lu\n", max_d, band_size); best_m = -1; min_k = 0; max_k = 0; d_path_idx = 0; max_idx = 0; for (d = 0; d < max_d; d ++ ) { if (max_k - min_k > band_size) { break; } for (k = min_k; k <= max_k; k += 2) { if ( (k == min_k) || ((k != max_k) && (V[ k - 1 + k_offset ] < V[ k + 1 + k_offset])) ) { pre_k = k + 1; x = V[ k + 1 + k_offset]; } else { pre_k = k - 1; x = V[ k - 1 + k_offset] + 1; } y = x - k; d_path[d_path_idx].d = d; d_path[d_path_idx].k = k; d_path[d_path_idx].x1 = x; d_path[d_path_idx].y1 = y; while ( x < q_len && y < t_len && query_seq[x] == target_seq[y] ){ x++; y++; } d_path[d_path_idx].x2 = x; d_path[d_path_idx].y2 = y; d_path[d_path_idx].pre_k = pre_k; d_path_idx ++; V[ k + k_offset ] = x; U[ k + k_offset ] = x + y; if ( x + y > best_m) { best_m = x + y; } if ( x >= q_len || y >= t_len) { aligned = true; max_idx = d_path_idx; break; } } // For banding new_min_k = max_k; new_max_k = min_k; for (k2 = min_k; k2 <= max_k; k2 += 2) { if (U[ k2 + k_offset] >= best_m - band_tolerance ) { if ( k2 < new_min_k ) { new_min_k = k2; } if ( k2 > new_max_k ) { new_max_k = k2; } } } max_k = new_max_k + 1; min_k = new_min_k - 1; // For no banding // max_k ++; // min_k --; // For debuging // printf("min_max_k,d, %ld %ld %ld\n", min_k, max_k, d); if (aligned == true) { align_rtn._qry_end = x; align_rtn._tgt_end = y; align_rtn._dist = d; align_rtn._size = (x + y + d) / 2; align_rtn._qry_bgn = 0; align_rtn._tgt_bgn = 0; d_path_sort(d_path, max_idx); //print_d_path(d_path, max_idx); if (get_aln_str) { cd = d; ck = k; aln_path_idx = 0; while (cd >= 0 && aln_path_idx < q_len + t_len + 1) { d_path_aux = (d_path_data2 *) get_dpath_idx( cd, ck, max_idx, d_path); aln_path[aln_path_idx].x = d_path_aux -> x2; aln_path[aln_path_idx].y = d_path_aux -> y2; aln_path_idx ++; aln_path[aln_path_idx].x = d_path_aux -> x1; aln_path[aln_path_idx].y = d_path_aux -> y1; aln_path_idx ++; ck = d_path_aux -> pre_k; cd -= 1; } aln_path_idx --; cx = aln_path[aln_path_idx].x; cy = aln_path[aln_path_idx].y; align_rtn._qry_bgn = cx; align_rtn._tgt_bgn = cy; aln_pos = 0; while ( aln_path_idx > 0 ) { aln_path_idx --; nx = aln_path[aln_path_idx].x; ny = aln_path[aln_path_idx].y; if (cx == nx && cy == ny){ continue; } if (nx == cx && ny != cy){ //advance in y for (i = 0; i < ny - cy; i++) { align_rtn._qry_aln_str[aln_pos + i] = '-'; } for (i = 0; i < ny - cy; i++) { align_rtn._tgt_aln_str[aln_pos + i] = target_seq[cy + i]; } aln_pos += ny - cy; } else if (nx != cx && ny == cy){ //advance in x for (i = 0; i < nx - cx; i++) { align_rtn._qry_aln_str[aln_pos + i] = query_seq[cx + i]; } for (i = 0; i < nx - cx; i++) { align_rtn._tgt_aln_str[aln_pos + i] = '-'; } aln_pos += nx - cx; } else { for (i = 0; i < nx - cx; i++) { align_rtn._qry_aln_str[aln_pos + i] = query_seq[cx + i]; } for (i = 0; i < ny - cy; i++) { align_rtn._tgt_aln_str[aln_pos + i] = target_seq[cy + i]; } aln_pos += ny - cy; } cx = nx; cy = ny; } align_rtn._size = aln_pos; } break; } } free(V); free(U); free(d_path); free(aln_path); return aligned; } } canu-1.3/src/utgcns/libNDFalcon/dw.H000066400000000000000000000112021272335422300172320ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2015-DEC-28 * are a 'United States Government Work', and * are released in the public domain * * Brian P. Walenz beginning on 2016-JAN-04 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ /* * ===================================================================================== * * Filename: dw.h * * Description: Common delclaration for the code base * * Version: 0.1 * Created: 07/16/2013 07:46:23 AM * Revision: none * Compiler: gcc * * Author: Jason Chin, * Company: * * ===================================================================================== #################################################################################$$ # Copyright (c) 2011-2014, Pacific Biosciences of California, Inc. # # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted (subject to the limitations in the # disclaimer below) provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # # * Neither the name of Pacific Biosciences nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE # GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC # BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED # WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. #################################################################################$$ */ #ifndef NDALIGNER_H #define NDALIGNER_H #include #include namespace NDalignment { typedef int32_t seq_coor_t; class NDalignResult { public: NDalignResult() { clear(); _qry_aln_str = _tgt_aln_str = 0; }; ~NDalignResult() { std::free(_qry_aln_str); std::free(_tgt_aln_str); }; void clear(void) { _qry_bgn = 0; _qry_end = 0; _tgt_bgn = 0; _tgt_end = 0; _dist = 0; _size = 0; }; int32_t _size; int32_t _dist; int32_t _qry_bgn; int32_t _qry_end; int32_t _tgt_bgn; int32_t _tgt_end; int32_t _olapLen; char* _qry_aln_str; char* _tgt_aln_str; }; typedef struct { seq_coor_t pre_k; seq_coor_t x1; seq_coor_t y1; seq_coor_t x2; seq_coor_t y2; } d_path_data; typedef struct { seq_coor_t d; seq_coor_t k; seq_coor_t pre_k; seq_coor_t x1; seq_coor_t y1; seq_coor_t x2; seq_coor_t y2; } d_path_data2; typedef struct { seq_coor_t x; seq_coor_t y; } path_point; bool align(const char *, seq_coor_t, const char *, seq_coor_t, seq_coor_t, bool, NDalignResult &aln); } #endif canu-1.3/src/utgcns/libNDalign/000077500000000000000000000000001272335422300164035ustar00rootroot00000000000000canu-1.3/src/utgcns/libNDalign/Binomial_Bound.C000066400000000000000000000053671272335422300214030ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2015-OCT-12 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "Binomial_Bound.H" #include "gkStore.H" #undef COMPUTE_IN_LOG_SPACE // Determined by EDIT_DIST_PROB_BOUND #define NORMAL_DISTRIB_THOLD 3.62 // Probability limit to "band" edit-distance calculation // Determines NORMAL_DISTRIB_THOLD #define EDIT_DIST_PROB_BOUND 1e-4 // Return the smallest n >= Start s.t. // prob [>= e errors in n binomial trials (p = error prob)] > EDIT_DIST_PROB_BOUND // int Binomial_Bound(int e, double p, int Start) { double Normal_Z, Mu_Power, Factorial, Poisson_Coeff; double q, Sum, P_Power, Q_Power, X; int k, n, Bin_Coeff, Ct; q = 1.0 - p; if (Start < e) Start = e; for (n = Start; n < AS_MAX_READLEN; n ++) { if (n <= 35) { Sum = 0.0; Bin_Coeff = 1; Ct = 0; P_Power = 1.0; Q_Power = pow (q, n); for (k = 0; k < e && 1.0 - Sum > EDIT_DIST_PROB_BOUND; k ++) { X = Bin_Coeff * P_Power * Q_Power; Sum += X; Bin_Coeff *= n - Ct; Bin_Coeff /= ++ Ct; P_Power *= p; Q_Power /= q; } if (1.0 - Sum > EDIT_DIST_PROB_BOUND) return(n); } else { Normal_Z = (e - 0.5 - n * p) / sqrt (n * p * q); if (Normal_Z <= NORMAL_DISTRIB_THOLD) return n; #ifndef COMPUTE_IN_LOG_SPACE Sum = 0.0; Mu_Power = 1.0; Factorial = 1.0; Poisson_Coeff = exp (- n * p); for (k = 0; k < e; k ++) { Sum += Mu_Power * Poisson_Coeff / Factorial; Mu_Power *= n * p; Factorial *= k + 1; } #else Sum = 0.0; Mu_Power = 0.0; Factorial = 0.0; Poisson_Coeff = - n * p; for (k = 0; k < e; k ++) { Sum += exp(Mu_Power + Poisson_Coeff - Factorial); Mu_Power += log(n * p); Factorial = lgamma(k + 1); } #endif if (1.0 - Sum > EDIT_DIST_PROB_BOUND) return(n); } } return(AS_MAX_READLEN); } canu-1.3/src/utgcns/libNDalign/Binomial_Bound.H000066400000000000000000000015621272335422300214010ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef BINOMIAL_BOUND_H #define BINOMIAL_BOUND_H #include "AS_global.H" int Binomial_Bound(int e, double p, int Start); #endif canu-1.3/src/utgcns/libNDalign/NDalgorithm-allocateMoreSpace.C000066400000000000000000000057511272335422300243100ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/utgcns/libNDalign/prefixEditDistance-allocateMoreSpace.C * * Modifications by: * * Brian P. Walenz from 2015-JUL-20 to 2015-AUG-05 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "NDalgorithm.H" // Allocate another block of 64mb for edits // Needs to be at least: // 52,432 to handle 40% error at 64k overlap // 104,860 to handle 80% error at 64k overlap // 209,718 to handle 40% error at 256k overlap // 419,434 to handle 80% error at 256k overlap // 3,355,446 to handle 40% error at 4m overlap // 6,710,890 to handle 80% error at 4m overlap // Bigger means we can assign more than one Edit_Array[] in one allocation. uint32 EDIT_SPACE_SIZE = 1 * 1024 * 1024; bool NDalgorithm::allocateMoreEditSpace(void) { // Determine the last allocated block, and the last assigned block int32 b = 0; // Last edit array assigned int32 e = 0; // Last edit array assigned more space int32 a = 0; // Last allocated block while (Edit_Array_Lazy[b] != NULL) b++; while (Edit_Space_Lazy[a] != NULL) a++; // Fill in the edit space array. Well, not quite yet. First, decide the minimum size. // // Element [0] can access from [-2] to [2] = 5 elements. // Element [1] can access from [-3] to [3] = 7 elements. // // Element [e] can access from [-2-e] to [2+e] = 5 + e * 2 elements // // So, our offset for this new block needs to put [e][0] at offset... int32 Offset = 2 + b; int32 Del = 6 + b * 2; int32 Size = EDIT_SPACE_SIZE; while (Size < Offset + Del) Size *= 2; // Allocate another block Edit_Space_Lazy[a] = new pedEdit [Size]; // And, now, fill in the edit space array. e = b; while ((Offset + Del < Size) && (e < Edit_Space_Max)) { Edit_Array_Lazy[e++] = Edit_Space_Lazy[a] + Offset; Offset += Del; Del += 2; } if (e == b) { fprintf(stderr, "Allocate_More_Edit_Space()-- ERROR: couldn't allocate enough space for even one more entry! e=%d\n", e); return(false); } assert(e != b); return(true); //fprintf(stderr, "WorkArea %d allocates space %d of size %d for array %d through %d\n", thread_id, a, Size, b, e-1); } canu-1.3/src/utgcns/libNDalign/NDalgorithm-extend.C000066400000000000000000000134751272335422300222160ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/utgcns/libNDalign/prefixEditDistance-extend.C * * Modifications by: * * Brian P. Walenz from 2015-JUL-20 to 2015-JUL-28 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-13 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "NDalgorithm.H" #undef DEBUG // See how far the exact match in Match extends. The match // refers to strings S and T with lengths S_Len and T_Len , // respectively. Set S_Lo , S_Hi , T_Lo and T_Hi to the // regions within S and T to which the match extends. // Return the type of overlap: NONE if doesn't extend to // the end of either fragment; LEFT_BRANCH_PT // or RIGHT_BRANCH_PT if match extends to the end of just one fragment; // or DOVETAIL if it extends to the end of both fragments, i.e., // it is a complete overlap. // Set Errors to the number of errors in the alignment if it is // a DOVETAIL overlap. pedOverlapType NDalgorithm::Extend_Alignment(Match_Node_t *Match, char *S, int32 S_Len, char *T, int32 T_Len, int32 &S_Lo, int32 &S_Hi, int32 &T_Lo, int32 &T_Hi) { int32 Leftover = 0; bool rMatchToEnd = true; bool lMatchToEnd = true; int32 S_Left_Begin = Match->Start - 1; int32 S_Right_Begin = Match->Start + Match->Len; int32 S_Right_Len = S_Len - S_Right_Begin; int32 T_Left_Begin = Match->Offset - 1; int32 T_Right_Begin = Match->Offset + Match->Len; int32 T_Right_Len = T_Len - T_Right_Begin; int32 Total_Olap = (min(Match->Start, Match->Offset) + Match->Len + min(S_Right_Len, T_Right_Len)); #ifdef DEBUG fprintf(stderr, "NDalgorithm::Extend_Alignment()-- S: %d-%d and %d-%d T: %d-%d and %d-%d\n", 0, S_Left_Begin, S_Right_Begin, S_Right_Begin + S_Right_Len, 0, T_Left_Begin, T_Right_Begin, T_Right_Begin + T_Right_Len); #endif Right_Score = 0; Right_Delta_Len = 0; Left_Score = 0; Left_Delta_Len = 0; bool invertLeftDeltas = false; bool invertRightDeltas = false; if ((S_Right_Len == 0) || (T_Right_Len == 0)) { S_Hi = 0; T_Hi = 0; rMatchToEnd = true; } else if (S_Right_Len <= T_Right_Len) { #ifdef DEBUG fprintf(stderr, "NDalgorithm::Extend_Alignment()-- FORWARD S T\n"); #endif forward(S + S_Right_Begin, S_Right_Len, T + T_Right_Begin, T_Right_Len, S_Hi, T_Hi, rMatchToEnd); for (int32 i=0; i 0) { if (Right_Delta[0] > 0) Left_Delta[Left_Delta_Len++] = -(Right_Delta[0] + Leftover + Match->Len); else Left_Delta[Left_Delta_Len++] = -(Right_Delta[0] - Leftover - Match->Len); } // WHY?! Does this mean the invesion on the forward() calls is backwards? // But note interaction with the if test just above here! for (int32 i=1; i0; k--) { assert(Edit_Array_Lazy[k] != NULL); // Analyze cells at errors = k-1 for the maximum -- no analysis needed, since we stored this cell as fromd. int32 from = Edit_Array_Lazy[k][d].fromd; int32 lasts = Edit_Array_Lazy[k][d].score; //Edit_Array_Lazy[k-1][from].display(k-1, from); //fprintf(stderr, "NDalgorithm::Set_Right_Delta()-- k-1=%5d d=%5d from=%5d lastr=%5d - r=%5d s=%5d d=%5d\n", // k-1, d, from, lastr, // Edit_Array_Lazy[k-1][from].row, Edit_Array_Lazy[k-1][from].score, Edit_Array_Lazy[k-1][from].fromd); if (from == d - 1) { Delta_Stack[Right_Delta_Len++] = Edit_Array_Lazy[k-1][d-1].row - lastr - 1; d--; lastr = Edit_Array_Lazy[k-1][from].row; } else if (from == d + 1) { Delta_Stack[Right_Delta_Len++] = lastr - Edit_Array_Lazy[k-1][d+1].row; d++; lastr = Edit_Array_Lazy[k-1][from].row; } else { //fprintf(stderr, "LeftDelta: mismatch at %d max=%d last=%d\n", maxr - lastr - 1, maxr, lastr); } } Delta_Stack[Right_Delta_Len++] = lastr + 1; for (int32 k=0, i=Right_Delta_Len-1; i>0; i--) Right_Delta[k++] = abs(Delta_Stack[i]) * Sign(Delta_Stack[i-1]); Right_Delta_Len--; } // Return the minimum number of changes (inserts, deletes, replacements) // needed to match string A[0 .. (m-1)] with a prefix of string // T[0 .. (n-1)] if it's not more than Error_Limit . // If no match, return the number of errors for the best match // up to a branch point. // Put delta description of alignment in Right_Delta and set // Right_Delta_Len to the number of entries there if it's a complete // match. // Set A_End and T_End to the rightmost positions where the // alignment ended in A and T , respectively. // Set Match_To_End true if the match extended to the end // of at least one string; otherwise, set it false to indicate // a branch point. void NDalgorithm::forward(char *A, int32 Alen, char *T, int32 Tlen, int32 &A_End, int32 &T_End, bool &Match_To_End) { assert (Alen <= Tlen); int32 Best_d = 0; int32 Best_e = 0; int32 Best_row = 0; int32 Best_score = 0; int32 Row = 0; int32 Dst = 0; int32 Err = 0; int32 Sco = 0; int32 fromd = 0; // Skip ahead over matches. The original used to also skip if either sequence was N. while ((Row < Alen) && (isMatch(A[Row], T[Row]))) { Sco += matchScore(A[Row], T[Row]); Row++; } if (Edit_Array_Lazy[0] == NULL) allocateMoreEditSpace(); Edit_Array_Lazy[0][0].row = Row; Edit_Array_Lazy[0][0].dist = Dst; Edit_Array_Lazy[0][0].errs = 0; Edit_Array_Lazy[0][0].score = Sco; Edit_Array_Lazy[0][0].fromd = INT32_MAX; // Exact match? if (Row == Alen) { A_End = Alen; T_End = Alen; Match_To_End = true; Right_Score = Sco; Right_Delta_Len = 0; return; } int32 Left = 0; int32 Right = 0; int32 Max_Score = PEDMINSCORE; int32 Max_Score_Len = 0; int32 Max_Score_Best_d = 0; int32 Max_Score_Best_e = 0; for (int32 ei=1; ei <= Edit_Space_Max; ei++) { if (Edit_Array_Lazy[ei] == NULL) if (allocateMoreEditSpace() == false) { // FAIL return; } Left = MAX (Left - 1, -ei); Right = MIN (Right + 1, ei); //fprintf(stderr, "FORWARD ei=%d Left=%d Right=%d\n", ei, Left, Right); Edit_Array_Lazy[ei-1][Left - 1].init(); Edit_Array_Lazy[ei-1][Left ].init(); // Of note, [0][0] on the first iteration is not reset here. Edit_Array_Lazy[ei-1][Right ].init(); Edit_Array_Lazy[ei-1][Right + 1].init(); for (int32 d = Left; d <= Right; d++) { // A mismatch. { int32 aPos = (1 + Edit_Array_Lazy[ei-1][d].row) - 1; // -1 because we need to compare the base we are at, int32 tPos = (1 + Edit_Array_Lazy[ei-1][d].row) + d - 1; // not the base we will be at after the mismatch Row = 1 + Edit_Array_Lazy[ei-1][d].row; Dst = Edit_Array_Lazy[ei-1][d].dist + 1; Err = Edit_Array_Lazy[ei-1][d].errs + 1; fromd = d; // If positive, we have a pointer into valid sequence. If not, this mismatch // doesn't make sense, and the row/score are set to bogus values. if ((aPos >= 0) && (tPos >= 0)) { assert (aPos <= Alen); assert( tPos <= Tlen); assert(A[aPos] != T[tPos]); Sco = Edit_Array_Lazy[ei-1][d].score + mismatchScore(A[aPos], T[tPos]); } else { Sco = PEDMINSCORE; } } // Insert a gap in A. Check the other sequence to see if this is a zero-cost gap. Note // agreement with future value of Row and what is used in isMatch() below. { int32 tPos = 0 + Edit_Array_Lazy[ei-1][d-1].row + d; //assert(tPos >= 0); //assert(tPos < Tlen); if ((tPos >= 0) && (tPos <= Tlen)) { int32 gapCost = isFreeGap( T[tPos] ) ? PEDFREEGAP : PEDGAP; //if (gapCost == 0) // fprintf(stderr, "NDalgorithm::forward()-- free A gap for aPos=%d tPos=%d t=%c/%d\n", tPos - d, tPos, T[tPos], T[tPos]); if (Edit_Array_Lazy[ei-1][d-1].score + gapCost > Sco) { Row = Edit_Array_Lazy[ei-1][d-1].row; Dst = Edit_Array_Lazy[ei-1][d-1].dist + (gapCost == PEDFREEGAP) ? 0 : 0; Err = Edit_Array_Lazy[ei-1][d-1].errs + (gapCost == PEDFREEGAP) ? 0 : 0; Sco = Edit_Array_Lazy[ei-1][d-1].score + gapCost; fromd = d-1; } } } // Insert a gap in T. // Testcase test-st-ts shows this works. { int32 aPos = 1 + Edit_Array_Lazy[ei-1][d+1].row; //assert(aPos >= 0); //assert(aPos < Tlen); if ((aPos >= 0) && (aPos <= Alen)) { int32 gapCost = isFreeGap( A[aPos] ) ? 0 : PEDGAP; //if (gapCost == 0) // fprintf(stderr, "NDalgorithm::forward()-- free T gap for aPos=%d tPos=%d a=%c/%d\n", aPos, aPos + d, A[aPos], A[aPos]); if (Edit_Array_Lazy[ei-1][d+1].score + gapCost > Sco) { Row = 1 + Edit_Array_Lazy[ei-1][d+1].row; Dst = Edit_Array_Lazy[ei-1][d+1].dist + (gapCost == PEDFREEGAP) ? 0 : 1; Err = Edit_Array_Lazy[ei-1][d+1].errs + (gapCost == PEDFREEGAP) ? 0 : 1; Sco = Edit_Array_Lazy[ei-1][d+1].score + gapCost; fromd = d+1; } } } // If A or B is N, that isn't a mismatch. // If A is lowercase and T is uppercase, it's a match. // If A is lowercase and T doesn't match, ignore the cost of the gap in B while ((Row < Alen) && (Row + d < Tlen) && (isMatch(A[Row], T[Row + d]))) { Sco += matchScore(A[Row], T[Row + d]); Row += 1; Dst += 1; Err += 0; } Edit_Array_Lazy[ei][d].row = Row; Edit_Array_Lazy[ei][d].dist = Dst; Edit_Array_Lazy[ei][d].errs = Err; Edit_Array_Lazy[ei][d].score = Sco; Edit_Array_Lazy[ei][d].fromd = fromd; //fprintf(stderr, "SET ei=%d d=%d -- row=%d dist=%d errs=%d score=%d fromd=%d\n", ei, d, Row, Dst, Err, Sco, fromd); if (Row == Alen || Row + d == Tlen) { A_End = Row; // One past last align position T_End = Row + d; Set_Right_Delta(ei, d); Match_To_End = true; return; //return(ei); } } // Over all diagonals. // Reset the band // // The .dist used to be .row. while ((Left <= Right) && (Left < 0) && (Edit_Array_Lazy[ei][Left].dist < Edit_Match_Limit[ Edit_Array_Lazy[ei][Left].errs ])) Left++; if (Left >= 0) while ((Left <= Right) && (Edit_Array_Lazy[ei][Left].dist + Left < Edit_Match_Limit[ Edit_Array_Lazy[ei][Left].errs ])) Left++; if (Left > Right) break; while ((Right > 0) && (Edit_Array_Lazy[ei][Right].dist + Right < Edit_Match_Limit[ Edit_Array_Lazy[ei][Right].errs ])) Right--; if (Right <= 0) while (Edit_Array_Lazy[ei][Right].dist < Edit_Match_Limit[ Edit_Array_Lazy[ei][Right].errs ]) Right--; assert (Left <= Right); for (int32 d = Left; d <= Right; d++) if (Edit_Array_Lazy[ei][d].score > Best_score) { Best_d = d; Best_e = ei; Best_row = Edit_Array_Lazy[ei][d].row; Best_score = Edit_Array_Lazy[ei][d].score; } if (Best_score > Max_Score) { Max_Score_Best_d = Best_d; Max_Score_Best_e = Best_e; Max_Score = Best_score; Max_Score_Len = Best_row; } } // Over all possible number of errors //fprintf(stderr, "NDalgorithm::forward()- iterated over all errors, return best found\n"); A_End = Max_Score_Len; T_End = Max_Score_Len + Max_Score_Best_d; Set_Right_Delta(Max_Score_Best_e, Max_Score_Best_d); Match_To_End = false; return; //return(Max_Score_Best_e); } canu-1.3/src/utgcns/libNDalign/NDalgorithm-reverse.C000066400000000000000000000261671272335422300224040ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/utgcns/libNDalign/prefixEditDistance-reverse.C * * Modifications by: * * Brian P. Walenz from 2015-JUL-20 to 2015-AUG-05 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-13 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "NDalgorithm.H" // Put the delta encoding of the alignment represented in Edit_Array // starting at row e (which is the number of errors) and column d // (which is the diagonal) and working back to the start, into // Left_Delta. Set Left_Delta_Len to the number of // delta entries and set leftover to the number of // characters that match after the last Left_Delta entry. // Don't allow the first delta to be an indel if it can be // converted to a substitution by adjusting t_end which // is where the alignment ends in the T string, which has length // t_len . // void NDalgorithm::Set_Left_Delta(int32 e, int32 d, int32 &leftover) { Left_Score = Edit_Array_Lazy[e][d].score; Left_Delta_Len = 0; int32 lastr = Edit_Array_Lazy[e][d].row; //fprintf(stderr, "NDalgorithm::Set_Left_Delta()-- e =%5d d=%5d lastr=%5d fromd=%5d\n", // e, d, lastr); for (int32 k=e; k>0; k--) { assert(Edit_Array_Lazy[k] != NULL); int32 from = Edit_Array_Lazy[k][d].fromd; int32 lasts = Edit_Array_Lazy[k][d].score; //Edit_Array_Lazy[k-1][from].display(k-1, from); //fprintf(stderr, "NDalgorithm::Set_Left_Delta()-- k-1=%5d d=%5d from=%5d lastr=%5d - r=%5d s=%5d d=%5d\n", // k-1, d, from, lastr, // Edit_Array_Lazy[k-1][from].row, Edit_Array_Lazy[k-1][from].score, Edit_Array_Lazy[k-1][from].fromd); if (from == d - 1) { Left_Delta[Left_Delta_Len++] = Edit_Array_Lazy[k-1][d-1].row - lastr - 1; d--; lastr = Edit_Array_Lazy[k-1][from].row; } else if (from == d + 1) { Left_Delta[Left_Delta_Len++] = lastr - Edit_Array_Lazy[k-1][d+1].row; d++; lastr = Edit_Array_Lazy[k-1][from].row; } else { //fprintf(stderr, "LeftDelta: mismatch at %d max=%d last=%d\n", maxr - lastr - 1, maxr, lastr); } } leftover = lastr; } // Return the minimum number of changes (inserts, deletes, replacements) // needed to match string A[0 .. (1-m)] right-to-left with a prefix of string // T[0 .. (1-n)] right-to-left if it's not more than Error_Limit . // If no match, return the number of errors for the best match // up to a branch point. // Put delta description of alignment in Left_Delta and set // Left_Delta_Len to the number of entries there. // Set A_End and T_End to the leftmost positions where the // alignment ended in A and T , respectively. // If the alignment succeeds set Leftover to the number of // characters that match after the last Left_Delta entry; // otherwise, set Leftover to zero. // Set Match_To_End true if the match extended to the end // of at least one string; otherwise, set it false to indicate // a branch point. void NDalgorithm::reverse(char *A, int32 Alen, // first sequence and length char *T, int32 Tlen, // second sequence and length int32 &A_End, int32 &T_End, int32 &Leftover, // <- novel bool &Match_To_End) { assert (Alen <= Tlen); int32 Best_d = 0; int32 Best_e = 0; int32 Best_row = 0; int32 Best_score = 0; int32 Row = 0; int32 Dst = 0; int32 Err = 0; int32 Sco = 0; int32 fromd = 0; // Skip ahead over matches. The original used to also skip if either sequence was N. while ((Row < Alen) && (isMatch(A[-Row], T[-Row]))) { Sco += matchScore(A[-Row], T[-Row]); Row++; } if (Edit_Array_Lazy[0] == NULL) allocateMoreEditSpace(); Edit_Array_Lazy[0][0].row = Row; Edit_Array_Lazy[0][0].dist = Dst; Edit_Array_Lazy[0][0].errs = 0; Edit_Array_Lazy[0][0].score = Sco; Edit_Array_Lazy[0][0].fromd = INT32_MAX; // Exact match? if (Row == Alen) { A_End = -Alen; T_End = -Alen; Leftover = Alen; Match_To_End = true; Left_Score = Sco; Left_Delta_Len = 0; return; } int32 Left = 0; int32 Right = 0; int32 Max_Score = PEDMINSCORE; int32 Max_Score_Len = 0; int32 Max_Score_Best_d = 0; int32 Max_Score_Best_e = 0; for (int32 ei=1; ei <= Edit_Space_Max; ei++) { if (Edit_Array_Lazy[ei] == NULL) if (allocateMoreEditSpace() == false) { // FAIL return; } Left = MAX (Left - 1, -ei); Right = MIN (Right + 1, ei); //fprintf(stderr, "REVERSE ei=%d Left=%d Right=%d\n", ei, Left, Right); Edit_Array_Lazy[ei-1][Left - 1].init(); Edit_Array_Lazy[ei-1][Left ].init(); // Of note, [0][0] on the first iteration is not reset here. Edit_Array_Lazy[ei-1][Right ].init(); Edit_Array_Lazy[ei-1][Right + 1].init(); for (int32 d = Left; d <= Right; d++) { // A mismatch. { int32 aPos = -(1 + Edit_Array_Lazy[ei-1][d].row) + 1; // +1 because we need to compare the base we are at, int32 tPos = -(1 + Edit_Array_Lazy[ei-1][d].row) - d + 1; // not the base we will be at after the mismatch Row = 1 + Edit_Array_Lazy[ei-1][d].row; Dst = Edit_Array_Lazy[ei-1][d].dist + 1; Err = Edit_Array_Lazy[ei-1][d].errs + 1; fromd = d; //fprintf(stderr, "aPos %d tPos %d\n", aPos, tPos); // If negative, we have a pointer into valid sequence. If not, this mismatch // doesn't make sense, and the row/score are set to bogus values. if ((aPos <= 0) && (tPos <= 0)) { assert(-aPos <= Alen); assert(-tPos <= Tlen); assert(A[aPos] != T[tPos]); Sco = Edit_Array_Lazy[ei-1][d].score + mismatchScore(A[aPos], T[tPos]); } else { Sco = PEDMINSCORE; } } // Insert a gap in A. Check the other sequence to see if this is a zero-cost gap. Note // agreement with future value of Row and what is used in isMatch() below. // Testcase test-st-ts shows this works. { int32 tPos = -(0 + Edit_Array_Lazy[ei-1][d-1].row) - d; //assert( tPos <= 0); Not true at the lower end; we'll just skip the cell since it's invalid //assert(-tPos < Tlen); if ((tPos <= 0) && (-tPos < Tlen)) { int32 gapCost = isFreeGap( T[tPos] ) ? PEDFREEGAP : PEDGAP; //if (gapCost == 0) // fprintf(stderr, "NDalgorithm::reverse()-- free A gap for aPos=%d tPos=%d t=%c/%d\n", tPos + d, tPos, T[tPos], T[tPos]); if (Edit_Array_Lazy[ei-1][d-1].score + gapCost > Sco) { Row = Edit_Array_Lazy[ei-1][d-1].row; Dst = Edit_Array_Lazy[ei-1][d-1].dist + (gapCost == PEDFREEGAP) ? 0 : 0; Err = Edit_Array_Lazy[ei-1][d-1].errs + (gapCost == PEDFREEGAP) ? 0 : 0; Sco = Edit_Array_Lazy[ei-1][d-1].score + gapCost; fromd = d-1; } } } // Insert a gap in T. { int32 aPos = -(1 + Edit_Array_Lazy[ei-1][d+1].row); //assert( aPos <= 0); //assert(-aPos < Tlen); if ((aPos <= 0) && (-aPos < Alen)) { int32 gapCost = isFreeGap( A[aPos] ) ? 0 : PEDGAP; //if (gapCost == 0) // fprintf(stderr, "NDalgorithm::reverse()-- free T gap for aPos=%d tPos=%d t=%c/%d\n", aPos, aPos - d, A[aPos], A[aPos]); if (Edit_Array_Lazy[ei-1][d+1].score + gapCost > Sco) { Row = 1 + Edit_Array_Lazy[ei-1][d+1].row; Dst = Edit_Array_Lazy[ei-1][d+1].dist + (gapCost == PEDFREEGAP) ? 0 : 1; Err = Edit_Array_Lazy[ei-1][d+1].errs + (gapCost == PEDFREEGAP) ? 0 : 1; Sco = Edit_Array_Lazy[ei-1][d+1].score + gapCost; fromd = d+1; } } } // If A or B is N, that isn't a mismatch. // If A is lowercase and T is uppercase, it's a match. // If A is lowercase and T doesn't match, ignore the cost of the gap in B while ((Row < Alen) && (Row + d < Tlen) && (isMatch(A[-Row], T[-Row - d]))) { Sco += matchScore(A[-Row], T[-Row - d]); Row += 1; Dst += 1; Err += 0; } Edit_Array_Lazy[ei][d].row = Row; Edit_Array_Lazy[ei][d].dist = Dst; Edit_Array_Lazy[ei][d].errs = Err; Edit_Array_Lazy[ei][d].score = Sco; Edit_Array_Lazy[ei][d].fromd = fromd; if (Row == Alen || Row + d == Tlen) { A_End = - Row; // One past last align position T_End = - Row - d; Set_Left_Delta(ei, d, Leftover); Match_To_End = true; return; //return(ei); } } // Over all diagonals. // Reset the band // // The .dist used to be .row. while ((Left <= Right) && (Left < 0) && (Edit_Array_Lazy[ei][Left].dist < Edit_Match_Limit[ Edit_Array_Lazy[ei][Left].errs ])) Left++; if (Left >= 0) while ((Left <= Right) && (Edit_Array_Lazy[ei][Left].dist + Left < Edit_Match_Limit[ Edit_Array_Lazy[ei][Left].errs ])) Left++; if (Left > Right) break; while ((Right > 0) && (Edit_Array_Lazy[ei][Right].dist + Right < Edit_Match_Limit[ Edit_Array_Lazy[ei][Right].errs ])) Right--; if (Right <= 0) while (Edit_Array_Lazy[ei][Right].dist < Edit_Match_Limit[ Edit_Array_Lazy[ei][Right].errs ]) Right--; assert (Left <= Right); for (int32 d = Left; d <= Right; d++) if (Edit_Array_Lazy[ei][d].score > Best_score) { Best_d = d; Best_e = ei; Best_row = Edit_Array_Lazy[ei][d].row; Best_score = Edit_Array_Lazy[ei][d].score; } if (Best_score > Max_Score) { Max_Score_Best_d = Best_d; Max_Score_Best_e = Best_e; Max_Score = Best_score; Max_Score_Len = Best_row; } } // Over all possible number of errors //fprintf(stderr, "NDalgorithm::reverse()-- iterated over all errors, return best found\n"); A_End = - Max_Score_Len; T_End = - Max_Score_Len - Max_Score_Best_d; Set_Left_Delta(Max_Score_Best_e, Max_Score_Best_d, Leftover); Match_To_End = false; return; //return(Max_Score_Best_e); } canu-1.3/src/utgcns/libNDalign/NDalgorithm.C000066400000000000000000000127331272335422300207250ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/utgcns/libNDalign/prefixEditDistance.C * * Modifications by: * * Brian P. Walenz from 2015-JUL-20 to 2015-AUG-05 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-FEB-25 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "NDalgorithm.H" #include "Binomial_Bound.H" const char * toString(pedAlignType at) { const char *ret = NULL; switch (at) { case pedOverlap: ret = "full-overlap"; break; case pedLocal: ret = "local-overlap"; break; case pedGlobal: ret = "global-alignment"; break; default: assert(0); break; } return(ret); } const char * toString(pedOverlapType ot) { const char *ret = NULL; switch (ot) { case pedBothBranch: ret = "both-branch"; break; case pedLeftBranch: ret = "left-branch"; break; case pedRightBranch: ret = "right-branch"; break; case pedDovetail: ret = "dovetail-overlap"; break; default: assert(0); break; } return(ret); } NDalgorithm::NDalgorithm(pedAlignType alignType_, double maxErate_) { alignType = alignType_; maxErate = maxErate_; ERRORS_FOR_FREE = 1; MIN_BRANCH_END_DIST = 20; MIN_BRANCH_TAIL_SLOPE = ((maxErate > 0.06) ? 1.0 : 0.20); Left_Delta = new int32 [AS_MAX_READLEN]; Right_Delta = new int32 [AS_MAX_READLEN]; Delta_Stack = new int32 [AS_MAX_READLEN]; allocated = 3 * AS_MAX_READLEN * sizeof(int32); Edit_Space_Max = AS_MAX_READLEN; //(alignType == pedGlobal) ? (AS_MAX_READLEN) : (1 + (int32)ceil(maxErate * AS_MAX_READLEN)); Edit_Space_Lazy = new pedEdit * [Edit_Space_Max]; Edit_Array_Lazy = new pedEdit * [Edit_Space_Max]; memset(Edit_Space_Lazy, 0, sizeof(pedEdit *) * Edit_Space_Max); memset(Edit_Array_Lazy, 0, sizeof(pedEdit *) * Edit_Space_Max); allocated += Edit_Space_Max * sizeof (pedEdit *); allocated += Edit_Space_Max * sizeof (pedEdit *); int32 dataIndex = (int)ceil(maxErate * 100) - 1; if ((dataIndex < 0) || (50 <= dataIndex)) fprintf(stderr, "NDalgorithm()-- Invalid maxErate=%f -> dataIndex=%d\n", maxErate, dataIndex); assert(0 <= dataIndex); assert(dataIndex < 50); #if 0 // Use the precomputed values. { Edit_Match_Limit_Allocation = NULL; Edit_Match_Limit = Edit_Match_Limit_Data[dataIndex]; fprintf(stderr, "NDalgorithm()-- Set Edit_Match_Limit to %p; dataIndex=%d 6 = %p\n", Edit_Match_Limit, dataIndex, Edit_Match_Limit_0600); } #else // Compute values on the fly. { int32 MAX_ERRORS = (1 + (int32)ceil(maxErate * AS_MAX_READLEN)); Edit_Match_Limit_Allocation = new int32 [MAX_ERRORS + 1]; for (int32 e=0; e<= ERRORS_FOR_FREE; e++) Edit_Match_Limit_Allocation[e] = 0; int Start = 1; for (int32 e=ERRORS_FOR_FREE + 1; e= Edit_Match_Limit_Allocation[e-1]); } Edit_Match_Limit = Edit_Match_Limit_Allocation; } #endif for (int32 i=0; i <= AS_MAX_READLEN; i++) { //Error_Bound[i] = (int32) (i * maxErate + 0.0000000000001); Error_Bound[i] = (int32)ceil(i * maxErate); } // Value to add for a match in finding branch points. // // ALH: Note that maxErate also affects what overlaps get found // // ALH: Scoring seems to be unusual: given an alignment of length l with k mismatches, the score // seems to be computed as l + k * error value and NOT (l-k)*match+k*error // // I.e. letting x := DEFAULT_BRANCH_MATCH_VAL, the max mismatch fraction p to give a non-negative // score would be p = x/(1-x); conversely, to compute x for a goal p, we have x = p/(1+p). E.g. // // for p=0.06, x = .06 / (1.06) = .0566038 // for p=0.35, x = .35 / (1.35) = .259259 // for p=0.2, x = .20 / (1.20) = .166667 // for p=0.15, x = .15 / (1.15) = .130435 // // Value was for 6% vs 35% error discrimination. // // Converting to integers didn't make it faster. // // Corresponding error value is this value minus 1.0 Branch_Match_Value = maxErate / (1 + maxErate); for (uint32 ii=0; ii<256; ii++) tolower[ii] = ::tolower(ii); }; NDalgorithm::~NDalgorithm() { delete [] Left_Delta; delete [] Right_Delta; delete [] Delta_Stack; for (uint32 i=0; i 0) - ((a) < 0) ) // Scores used in the dynamic programming. // #define PEDMATCH 2 // Match from UPPERCASE to UPPERCASE #define PEDGAPMATCH 1 // Match from LOWERCASE to UPPERCASE #define PEDMISMATCH -3 // Mismatch #define PEDFREEGAP 1 // Gap from LOWERCASE #define PEDGAP -5 // Gap // PEDMINSCORE wants to be INT32_MIN, except that it will underflow, and wrap around to the highest // score possible, on the first insert. I think it suffices to set this just slightly higher than // the minimum, since it should never be picked as a maximum, and thus never see more than one // insert. Just to be safe, we give it lots of space. // #define PEDMINSCORE (INT32_MIN - PEDGAP * 32768) enum pedAlignType { pedOverlap = 0, // Former normal overlap pedLocal = 1, // Former partial overlap pedGlobal = 2 // New forced global overlap }; enum pedOverlapType { pedBothBranch = 0, pedLeftBranch = 1, pedRightBranch = 2, pedDovetail = 3 }; const char *toString(pedAlignType at); const char *toString(pedOverlapType ot); // the input to Extend_Alignment. class Match_Node_t { public: int32 Offset; // To start of exact match in hash-table frag int32 Len; // Of exact match int32 Start; // Of exact match in current (new) frag int32 Next; // Subscript of next match in list }; class pedEdit { public: int32 row; // Position in the A sequence; was previously the score too int32 dist; // The number of non-free bases in the alignment, used to decide band int32 errs; // The number of non-free errors in the alignment, used to decide band int32 score; // The dynamic programming score, used to decide best align int32 fromd; // For backtracking, where we came from void init(void) { row = -2; dist = -2; errs = 0; score = PEDMINSCORE; fromd = INT32_MAX; }; void display(int32 e, int32 d) { fprintf(stderr, "e=%d d=%d - ptr %p - ", e, d, this); fprintf(stderr, "row=%d ", row); fprintf(stderr, "dist=%d ", dist); fprintf(stderr, "errs=%d ", errs); fprintf(stderr, "score=%d ", score); fprintf(stderr, "fromd=%d\n", fromd); }; }; class NDalgorithm { public: NDalgorithm(pedAlignType alignType_, double maxErate_); ~NDalgorithm(); private: bool allocateMoreEditSpace(void); void Set_Right_Delta(int32 e, int32 d); void forward(char *A, int32 m, char *T, int32 n, int32 &A_End, int32 &T_End, bool &Match_To_End); void Set_Left_Delta(int32 e, int32 d, int32 &leftover); void reverse(char *A, int32 m, char *T, int32 n, int32 &A_End, int32 &T_End, int32 &Leftover, // <- novel bool &Match_To_End); // Returns true if letter 'a' from sequence A matches letter 't' from sequence T. // Wanted to allow lowercase as free matches, but the O(ND) algorithm doesn't support that. // bool isMatch(char a, char t) { return(a == t); }; // Returns the score of a match. Pretty basic, was written to support 'a' == 'A' matches, but // the O(ND) algorithm cannot support that. // int32 matchScore(char a, char t) { if (isMatch(a, t)) return(PEDMATCH); fprintf(stderr, "ERROR: a=%c/%d does not match t=%c/%d\n", a, a, t, t); assert(isMatch(a, t) == true); return(0); }; // Returns the score of a mismatch. As with gaps, we can get free mismatches to lower case letters, // which should prevent alignments such as: // // A: GtTTgaTGACCTGG (the scoring should be 2 mismatches then 3 gaps) // B: GTTTGA---CCTGG // // Favoring instead // // A: GtTTgaTGACCTGG (the scoring should be 2 free gaps, 1 gap and 2 matches) // B: GTTT---GACCTGG // int32 mismatchScore(char a, char t) { if ((a == tolower[a]) || (t == tolower[t])) return(PEDGAPMATCH); return(PEDMISMATCH); } // Returns true if the gap we'd be adding to T is zero cost: // lower case 'a' is a don't case position in sequence A // bool isFreeGap(char a) { if (a == 0) return(false); return(a == tolower[a]); } public: pedOverlapType Extend_Alignment(Match_Node_t *Match, char *S, int32 S_Len, char *T, int32 T_Len, int32 &S_Lo, int32 &S_Hi, int32 &T_Lo, int32 &T_Hi); int32 score(void) { return(Left_Score + Right_Score); }; public: // The three below were global #defines, two depended on the error rate which is now local. // The number of errors that are ignored in setting probability bound for terminating alignment // extensions in edit distance calculations uint32 ERRORS_FOR_FREE; // Branch points must be at least this many bases from the end of the fragment to be reported uint32 MIN_BRANCH_END_DIST; // Branch point tails must fall off from the max by at least this rate double MIN_BRANCH_TAIL_SLOPE; pedAlignType alignType; double maxErate; uint64 allocated; int32 Left_Score; int32 Left_Delta_Len; int32 *Left_Delta; int32 Right_Score; int32 Right_Delta_Len; int32 *Right_Delta; int32 *Delta_Stack; int32 Edit_Space_Max; pedEdit **Edit_Space_Lazy; // Array of pointers, if set, it was a new'd allocation pedEdit **Edit_Array_Lazy; // Array of pointers, some are not new'd allocations // This array [e] is the minimum value of Edit_Array[e][d] // to be worth pursuing in edit-distance computations between reads const int32 *Edit_Match_Limit; int32 *Edit_Match_Limit_Allocation; // The maximum number of errors allowed in a match between reads of length i, // which is i * AS_OVL_ERROR_RATE. int32 Error_Bound[AS_MAX_READLEN + 1]; // Scores of matches and mismatches in alignments. Alignment ends at maximum score. double Branch_Match_Value; char tolower[256]; }; #endif // NDALGORITHM_H canu-1.3/src/utgcns/libNDalign/NDalign.C000066400000000000000000000727161272335422300200400ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/overlapInCore/overlapAlign.C * * Modifications by: * * Brian P. Walenz from 2015-JUN-17 to 2015-AUG-25 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-13 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "NDalign.H" #include "kMer.H" #include "merStream.H" #include "stddev.H" #include "Display_Alignment.H" #undef DEBUG_ALGORITHM // Some details. #undef DEBUG_HITS // Lots of details (chainHits()) #undef SEED_NON_OVERLAPPING // Allow mismatches in seeds #define SEED_OVERLAPPING #define DISPLAY_WIDTH 250 NDalign::NDalign(pedAlignType alignType, double maxErate, int32 merSize) { _alignType = alignType; _maxErate = maxErate; _merSizeInitial = merSize; _aID = UINT32_MAX; _aStr = NULL; _aLen = 0; _aLoOrig = 0; _aHiOrig = 0; _bID = UINT32_MAX; _bStr = NULL; _bLen = 0; _bLoOrig = 0; _bHiOrig = 0; _bFlipped = false; _bRevMax = 0; _bRev = NULL; _editDist = new NDalgorithm(_alignType, _maxErate); _minDiag = 0; _maxDiag = 0; _merSize = 0; _topDisplay = NULL; _botDisplay = NULL; _resDisplay = NULL; // Initialize Constants for (uint32 ii=0; ii<256; ii++) { acgtToBit[ii] = 0x00; acgtToVal[ii] = 0x03; } // Do NOT include lowercase letters here. These are 'don't care' matches // in consensus, and are usually matches we want to be skipping. // // ATGtTTgaTGACC vs ATGtTTgaTGACC // ATG-TT--TGACC ATGTTT---GACC // // While, yes, the second form is simpler (and would be correct had the last T been // optional) the first is what consensus is expecting to find. #if 0 acgtToBit['a'] = acgtToBit['A'] = 0x00; // Bit encoding of ACGT acgtToBit['c'] = acgtToBit['C'] = 0x01; acgtToBit['g'] = acgtToBit['G'] = 0x02; acgtToBit['t'] = acgtToBit['T'] = 0x03; acgtToVal['a'] = acgtToVal['A'] = 0x00; // Word is valid if zero acgtToVal['c'] = acgtToVal['C'] = 0x00; acgtToVal['g'] = acgtToVal['G'] = 0x00; acgtToVal['t'] = acgtToVal['T'] = 0x00; #else acgtToBit['A'] = 0x00; // Bit encoding of ACGT acgtToBit['C'] = 0x01; acgtToBit['G'] = 0x02; acgtToBit['T'] = 0x03; acgtToVal['A'] = 0x00; // Word is valid if zero acgtToVal['C'] = 0x00; acgtToVal['G'] = 0x00; acgtToVal['T'] = 0x00; #endif merMask[0] = 0x0000000000000000llu; merMask[1] = 0x0000000000000003llu; for (uint32 ii=2; ii<33; ii++) merMask[ii] = (merMask[ii-1] << 2) | 0x03; assert(merMask[ 6] == 0x0000000000000fffllu); assert(merMask[17] == 0x00000003ffffffffllu); assert(merMask[26] == 0x000fffffffffffffllu); assert(merMask[32] == 0xffffffffffffffffllu); } NDalign::~NDalign() { delete _editDist; delete [] _bRev; delete [] _topDisplay; delete [] _botDisplay; delete [] _resDisplay; } void NDalign::initialize(uint32 aID, char *aStr, int32 aLen, int32 aLo, int32 aHi, uint32 bID, char *bStr, int32 bLen, int32 bLo, int32 bHi, bool bFlipped) { #ifdef DEBUG_ALGORITHM fprintf(stderr, "NDalign::initialize()-- A %u %d-%d B %u %d-%d\n", aID, aLo, aHi, bID, bLo, bHi); #endif _aID = aID; _aStr = aStr; _aLen = aLen; _aLoOrig = aLo; _aHiOrig = aHi; _bID = bID; _bStr = bStr; _bLen = bLen; _bLoOrig = bLo; _bHiOrig = bHi; _bFlipped = bFlipped; if (_bFlipped == true) { if (_bRevMax < _bLen) { delete [] _bRev; _bRevMax = _bLen + 1000; _bRev = new char [_bRevMax]; } memcpy(_bRev, bStr, sizeof(char) * (_bLen + 1)); reverseComplementSequence(_bRev, _bLen); _bStr = _bRev; _bLoOrig = _bLen - bLo; // Now correct for the reverse complemented sequence _bHiOrig = _bLen - bHi; } assert(_aLoOrig < _aHiOrig); assert(_bLoOrig < _bHiOrig); //_editDist doesn't need to be cleared. _minDiag = 0; _maxDiag = 0; _merSize = 0; _aMap.clear(); _bMap.clear(); _rawhits.clear(); _hits.clear(); _hitr = UINT32_MAX; _bestResult.clear(); } // Set the min/max diagonal we will accept seeds for. It's just the min/max diagonal for the // two endpoints extended by half the erate times the align length. // // Returns false if there is no chance of an overlap - the computed alignment length is too small. // // The adjusts change the sequence start and end position (bgn + bgnAdj, end - endAdj), // and are intended to compensate for supplying sequences larger than absolutely necessary // (for when the exact boundaries aren't known). bool NDalign::findMinMaxDiagonal(int32 minLength, uint32 AbgnAdj, uint32 AendAdj, uint32 BbgnAdj, uint32 BendAdj) { _minDiag = 0; _maxDiag = 0; int32 aALen = (_aHiOrig - AendAdj) - (_aLoOrig + AbgnAdj); int32 bALen = (_bHiOrig - BendAdj) - (_bLoOrig + BbgnAdj); int32 alignLen = (aALen < bALen) ? bALen : aALen; if (alignLen < minLength) return(false); int32 bgnDiag = (_aLoOrig + AbgnAdj) - (_bLoOrig + BbgnAdj); int32 endDiag = (_aHiOrig - AendAdj) - (_bHiOrig - BendAdj); if (bgnDiag < endDiag) { _minDiag = bgnDiag - _maxErate * alignLen / 2; _maxDiag = endDiag + _maxErate * alignLen / 2; } else { _minDiag = endDiag - _maxErate * alignLen / 2; _maxDiag = bgnDiag + _maxErate * alignLen / 2; } // For very very short overlaps (mhap kindly reports 4 bp overlaps) reset the min/max // to something a little more permissive. //if (_minDiag > -5) _minDiag = -5; //if (_maxDiag < 5) _maxDiag = 5; if (_minDiag > _maxDiag) fprintf(stderr, "NDalign::findMinMaxDiagonal()-- ERROR: _minDiag=%d >= _maxDiag=%d\n", _minDiag, _maxDiag); assert(_minDiag <= _maxDiag); #ifdef DEBUG_ALGORITHM fprintf(stderr, "NDalign::findMinMaxDiagonal()-- A: _aLoOrig %d + AbgnAdj %d -- _aHiOrig %d - AendAdj %d\n", _aLoOrig, AbgnAdj, _aHiOrig, AendAdj); fprintf(stderr, "NDalign::findMinMaxDiagonal()-- B: _bLoOrig %d + BbgnAdj %d -- _bHiOrig %d - BendAdj %d\n", _bLoOrig, BbgnAdj, _bHiOrig, BendAdj); fprintf(stderr, "NDalign::findMinMaxDiagonal()-- min %d max %d -- span %d -- alignLen %d\n", _minDiag, _maxDiag, _maxDiag-_minDiag, alignLen); #endif return(true); } void NDalign::fastFindMersA(bool dupIgnore) { uint64 mer = 0x0000000000000000llu; uint64 val = 0xffffffffffffffffllu; uint64 mask = merMask[_merSize]; // Restict the mers we seed with the those in the overlap (plus a little wiggle room). If this // isn't done, overlaps in repeats are sometimes lost. In the A read, mers will drop out (e.g., // if a repeat at both the 5' and 3' end, and we have an overlap on one end only). In the B // read, the mers can now be on the wrong diagonal. int32 bgn = _aLoOrig - _merSize - _merSize; int32 end = _aHiOrig + _merSize; if (bgn < 0) bgn = 0; // Create mers. Since 'val' was initialized as invalid until the first _merSize things // are pushed on, no special case is needed to load the mer. It costs us two extra &'s // and the test for saving the valid mer while we initialize. for (int32 seqpos=bgn; (seqpos < end) && (_aStr[seqpos] != 0); seqpos++) { mer <<= 2; val <<= 2; mer |= acgtToBit[_aStr[seqpos]]; val |= acgtToVal[_aStr[seqpos]]; mer &= merMask[_merSize]; val &= merMask[_merSize]; if (val != 0x0000000000000000) // Not a valid mer. continue; // +1 - consider a 1-mer. The first time through we have a valid mer, but seqpos == 0. // To get an aMap position of zero (the true position) we need to add one. if (_aMap.find(mer) != _aMap.end()) { if (dupIgnore == true) _aMap[mer] = INT32_MAX; // Duplicate mer, now ignored! } else { _aMap[mer] = seqpos + 1 - _merSize; } } //fprintf(stderr, "Found %u hits in A at mersize %u dupIgnore %u t %u %u\n", _aMap.size(), _merSize, dupIgnore, t[0], t[1]); } void NDalign::fastFindMersB(bool dupIgnore) { uint64 mer = 0x0000000000000000llu; uint64 val = 0xffffffffffffffffllu; uint64 mask = merMask[_merSize]; // Like the A read, we limit to mers in the overlap region. int32 bgn = _bLoOrig - _merSize - _merSize; int32 end = _bHiOrig + _merSize; if (bgn < 0) bgn = 0; // Create mers. Since 'val' was initialized as invalid until the first _merSize things // are pushed on, no special case is needed to load the mer. It costs us two extra &'s // and the test for saving the valid mer while we initialize. for (int32 seqpos=bgn; (seqpos < end) && (_bStr[seqpos] != 0); seqpos++) { mer <<= 2; val <<= 2; mer |= acgtToBit[_bStr[seqpos]]; val |= acgtToVal[_bStr[seqpos]]; mer &= merMask[_merSize]; val &= merMask[_merSize]; if (val != 0x0000000000000000) // Not a valid mer. continue; if (_aMap.find(mer) == _aMap.end()) // Not in the A sequence, don't care. continue; int32 apos = _aMap[mer]; int32 bpos = seqpos + 1 - _merSize; if (apos == INT32_MAX) // Exists too many times in aSeq, don't care. continue; if ((apos - bpos < _minDiag) || (apos - bpos > _maxDiag)) // Too different. continue; if (_bMap.find(mer) != _bMap.end()) { if (dupIgnore == true) _bMap[mer] = INT32_MAX; // Duplicate mer, now ignored! } else { _bMap[mer] = bpos; } } //fprintf(stderr, "Found %u hits in B at mersize %u dupIgnore %u t %u %u %u %u %u\n", _bMap.size(), _merSize, dupIgnore, t[0], t[1], t[2], t[3], t[4]); } // Find seeds - hash the kmer and position from the first read, then lookup // each kmer in the second read. For unique hits, save the diagonal. Then what? // If the diagonal is too far from the expected diagonal (based on the overlap), // ignore the seed. // // If dupIgnore == true, ignore duplicates. Otherwise, use the first occurrence // // Returns false if no mers found. // bool NDalign::findSeeds(bool dupIgnore) { _merSize = _merSizeInitial; // Find mers in A findMersAgain: fastFindMersA(dupIgnore); if (_aMap.size() == 0) { _aMap.clear(); _bMap.clear(); _merSize--; if ((_merSize < 8) && (dupIgnore == true)) { _merSize = _merSizeInitial + 2; dupIgnore = false; } if (_merSize >= 8) goto findMersAgain; } // Find mers in B fastFindMersB(dupIgnore); if (_bMap.size() == 0) { _aMap.clear(); _bMap.clear(); _merSize--; if ((_merSize < 8) && (dupIgnore == true)) { _merSize = _merSizeInitial + 2; dupIgnore = false; } if (_merSize >= 8) goto findMersAgain; } // Still zero? Didn't find any unique seeds anywhere. if (_bMap.size() == 0) { #ifdef DEBUG_ALGORITHM fprintf(stderr, "NDalign::findSeeds()-- No seeds found.\n"); #endif return(false); } #ifdef DEBUG_ALGORITHM fprintf(stderr, "NDalign::findSeeds()-- Found %u seeds.\n", _bMap.size()); #endif return(true); } // For unique mers in B, if the mer is also unique in A, add a hit. bool NDalign::findHits(void) { for (map::iterator bit=_bMap.begin(); bit != _bMap.end(); bit++) { uint64 kmer = bit->first; int32 bpos = bit->second; if (bpos == INT32_MAX) // Exists too many times in bSeq, don't care about it. continue; int32 apos = _aMap[kmer]; assert(apos != INT32_MAX); // Should never get a bMap if the aMap isn't set if ((apos - bpos < _minDiag) || (apos - bpos > _maxDiag)) fprintf(stderr, "NDalign::findHits()-- kmer "F_X64" apos - bpos = %d _minDiag = %d _maxDiag = %d\n", kmer, apos-bpos, _minDiag, _maxDiag); assert(apos - bpos >= _minDiag); // ...these too. assert(apos - bpos <= _maxDiag); _rawhits.push_back(exactMatch(apos, bpos, _merSize)); } #ifdef DEBUG_ALGORITHM fprintf(stderr, "NDalign::findHits()-- Found %u hits.\n", _rawhits.size()); #endif return(true); } bool NDalign::chainHits(void) { // Sort by aPos (actually by length, then by aPos, but length is constant here). sort(_rawhits.begin(), _rawhits.end()); #ifdef DEBUG_HITS for (uint32 rr=0; rr<_rawhits.size(); rr++) if (_rawhits[rr].aBgn - _rawhits[rr].bBgn == 2289) fprintf(stderr, "NDalign::chainHits()-- HIT: %d - %d diag %d\n", _rawhits[rr].aBgn, _rawhits[rr].bBgn, _rawhits[rr].aBgn - _rawhits[rr].bBgn); #endif // Chain the _hits. This chains hits that are on the same diagonal and contiguous. _hits.push_back(_rawhits[0]); for (uint32 rr=1; rr<_rawhits.size(); rr++) { uint32 hh = _hits.size() - 1; bool merge = false; assert(_rawhits[rr-1].aBgn < _rawhits[rr].aBgn); #ifdef SEED_NON_OVERLAPPING // Allows non-overlapping (mismatch only) hits - but breaks seeding of alignment // --------- end of next hit ----------- ----- end of existing hit ----- int32 da = _rawhits[rr].aBgn + _rawhits[rr].tLen - _hits[hh].aBgn - _hits[hh].tLen; int32 db = _rawhits[rr].bBgn + _rawhits[rr].tLen - _hits[hh].bBgn - _hits[hh].tLen; assert(da > 0); if (da == db) merge = true; #endif #ifdef SEED_OVERLAPPING // Requires overlapping hits - full block of identity // - start of next - ------- end of existing ------- int32 da = _rawhits[rr].aBgn - _hits[hh].aBgn - _hits[hh].tLen; int32 db = _rawhits[rr].bBgn - _hits[hh].bBgn - _hits[hh].tLen; if ((da < 0) && (da == db)) merge = true; #endif if (merge) { //fprintf(stderr, "NDalign::chainHits()-- MERGE HIT: %d - %d diag %d da=%d db=%d\n", _rawhits[rr].aBgn, _rawhits[rr].bBgn, _rawhits[rr].aBgn - _rawhits[rr].bBgn, da, db); _hits[hh].tLen = _rawhits[rr].aBgn + _rawhits[rr].tLen - _hits[hh].aBgn; } else { //fprintf(stderr, "NDalign::chainHits()-- NEW HIT: %d - %d diag %d da=%d db=%d\n", _rawhits[rr].aBgn, _rawhits[rr].bBgn, _rawhits[rr].aBgn - _rawhits[rr].bBgn, da, db); _hits.push_back(_rawhits[rr]); } } // Sort by longest sort(_hits.begin(), _hits.end()); #ifdef DEBUG_HITS for (uint32 hh=0; hh<_hits.size(); hh++) { fprintf(stderr, "NDalign::chainHits()-- hit %02u %5d-%5d diag %d len %3u\n", hh, _hits[hh].aBgn, _hits[hh].bBgn, _hits[hh].aBgn - _hits[hh].bBgn, _hits[hh].tLen); } #endif #ifdef DEBUG_ALGORITHM fprintf(stderr, "NDalign::chainHits()-- Found %u chains of hits.\n", _hits.size()); #endif return(true); } bool NDalign::makeNullHit(void) { _hits.push_back(exactMatch(0, 0, 0)); return(true); } bool NDalign::processHits(void) { // If the first time here, set the hit iterator to zero, otherwise move to the next one. // And then return if there are no more hits to iterate over. if (_hitr == UINT32_MAX) _hitr = 0; else _hitr++; if (_hitr >= _hits.size()) return(false); // While hits, process them. // // If a good hit is found, return, leaving hitr as is. The next time we enter this function, // we'll increment hitr and process the next hit. If no good hit is found, we iterate the loop // until a good one is found, or we run out of hits. for (; _hitr < _hits.size(); _hitr++) { Match_Node_t match; match.Start = _hits[_hitr].aBgn; // Begin position in a match.Offset = _hits[_hitr].bBgn; // Begin position in b match.Len = _hits[_hitr].tLen; // tLen can include mismatches if alternate scoring is used! match.Next = 0; // Not used here #ifdef SEED_NON_OVERLAPPING match.Offset = _merSize; // Really should track this in the hits, oh well. #endif #ifdef DEBUG_ALGORITHM fprintf(stderr, "NDalign::processHits()--\n"); fprintf(stderr, "NDalign::processHits()-- Extend_Alignment Astart %d Bstart %d length %d\n", match.Start, match.Offset, match.Len); fprintf(stderr, "NDalign::processHits()--\n"); fprintf(stderr, ">A len=%u\n", _aLen); fwrite(_aStr, sizeof(char), _aLen, stderr); fprintf(stderr, "\n"); fprintf(stderr, ">B len=%u\n", _bLen); fwrite(_bStr, sizeof(char), _bLen, stderr); fprintf(stderr, "\n"); #endif int32 aLo=0, aHi=0; int32 bLo=0, bHi=0; char origScore[1024]; pedOverlapType olapType = _editDist->Extend_Alignment(&match, // Initial exact match, relative to start of string _aStr, _aLen, _bStr, _bLen, aLo, aHi, // Output: Regions which the match extends bLo, bHi); aHi++; // Add one to the end point because Extend_Alignment returns the base-based coordinate. bHi++; // Is this a better overlap than what we have? Save it and update statistics. if (((score() < _editDist->score())) || ((score() <= _editDist->score()) && (length() > ((aHi - aLo) + (bHi - bLo) + _editDist->Left_Delta_Len) / 2))) { #ifdef DEBUG_ALGORITHM fprintf(stderr, "NDalign::processHits()--\n"); fprintf(stderr, "NDalign::processHits()-- %s\n", (length() > 0) ? "Save better alignment" : "First alignment"); fprintf(stderr, "NDalign::processHits()--\n"); sprintf(origScore, "NDalign::processHits()-- OLD length %u erate %f score %u (%d-%d %d-%d)\n", length(), erate(), score(), abgn(), aend(), bbgn(), bend()); #endif _bestResult.save(aLo, aHi, bLo, bHi, _editDist->score(), olapType, _editDist->Left_Delta_Len, _editDist->Left_Delta); display("NDalign::processHits()-- ", false); _bestResult.setErate(1.0 - (double)(_matches + _gapmatches) / (length() - _freegaps)); #ifdef DEBUG_ALGORITHM fprintf(stderr, "%sNDalign::processHits()-- NEW length %u erate %f score %u (%d-%d %d-%d)\n", origScore, length(), erate(), score(), abgn(), aend(), bbgn(), bend()); #endif } else { olapType = pedBothBranch; #ifdef DEBUG_ALGORITHM fprintf(stderr, "NDalign::processHits()-- DON'T save alignment - OLD length %u erate %f score %u (%d-%d %d-%d) ", length(), erate(), score(), abgn(), aend(), bbgn(), bend()); fprintf(stderr, "NDalign::processHits()-- NEW length %u score %u coords %u-%u %u-%u\n", ((aHi - aLo) + (bHi - bLo) + _editDist->Left_Delta_Len) / 2, _editDist->score(), aLo, aHi, bLo, bHi); #endif } // If a dovetail, we're done. Let the client figure out if the quality is good. if (olapType == pedDovetail) return(true); } // Over all seeds. // No more seeds to align. Did we save an alignment? return(score() > 0); } #define ABS(x) (((x) < 0) ? -(x) : (x)) // A simple scan for blocks of gaps. The delta values here are relative to the last gap, // so a block of too much gap will have N delta values with small values. // bool NDalign::scanDeltaForBadness(bool verbose, bool showAlign) { double ema = 0.0; double alpha = 0.001; // Smaller will increase the averaging length int32 badBlocks = 0; for (uint32 ii=0; _resDisplay[ii]; ii++) { if (_resDisplay[ii] == '^') ema = computeExponentialMovingAverage(alpha, ema, 1.0); else ema = computeExponentialMovingAverage(alpha, ema, 0.0); if (ema > 0.25) badBlocks++; } if ((verbose == true) && (badBlocks > 0)) { fprintf(stderr, "NDalign::scanForDeltaBadness()-- Potential bad alignment: found %d bad blocks (alpha %f)\n", badBlocks, alpha); if (showAlign == true) display("NDalign::scanForDeltaBadness()-- ", true); } return(badBlocks > 0); } void NDalign::realignForward(bool displayAlgorithm, bool displayAlign) { Match_Node_t match; match.Start = abgn(); // Begin position in a match.Offset = bbgn(); // Begin position in b match.Len = 0; match.Next = 0; // Not used here int32 aLo=0, aHi=0; int32 bLo=0, bHi=0; char origScore[1024]; if (displayAlgorithm) fprintf(stderr, "NDalign::realignForward()--\n"); pedOverlapType olapType = _editDist->Extend_Alignment(&match, // Initial exact match, relative to start of string _aStr, _aLen, _bStr, _bLen, aLo, aHi, // Output: Regions which the match extends bLo, bHi); aHi++; // Add one to the end point because Extend_Alignment returns the base-based coordinate. bHi++; // Is this a better overlap than what we have? if (((score() < _editDist->score())) || ((score() <= _editDist->score()) && (length() > ((aHi - aLo) + (bHi - bLo) + _editDist->Left_Delta_Len) / 2))) { if (displayAlgorithm) { fprintf(stderr, "NDalign::realignForward()--\n"); fprintf(stderr, "NDalign::realignForward()-- Save better alignment\n"); fprintf(stderr, "NDalign::realignForward()--\n"); sprintf(origScore, "NDalign::realignForward()-- OLD length %u erate %f score %u (%d-%d %d-%d)\n", length(), erate(), score(), abgn(), aend(), bbgn(), bend()); } _bestResult.save(aLo, aHi, bLo, bHi, _editDist->score(), olapType, _editDist->Left_Delta_Len, _editDist->Left_Delta); display("NDalign::realignForward()-- ", displayAlign); _bestResult.setErate(1.0 - (double)(_matches + _gapmatches) / (length() - _freegaps)); if (displayAlgorithm) { fprintf(stderr, "%sNDalign::realignForward()-- NEW length %u erate %f score %u (%d-%d %d-%d)\n", origScore, length(), erate(), score(), abgn(), aend(), bbgn(), bend()); } } else if (displayAlgorithm) { fprintf(stderr, "NDalign::realignForward()-- Alignment no better - OLD length %u erate %f score %u (%d-%d %d-%d)\n", length(), erate(), score(), abgn(), aend(), bbgn(), bend()); fprintf(stderr, "NDalign::realignForward()-- Alignment no better - NEW length %u erate %f score %u (%d-%d %d-%d)\n", ((aHi - aLo) + (bHi - bLo) + _editDist->Left_Delta_Len) / 2, 0.0, _editDist->score(), aLo, aHi, bLo, bHi); //display("NDalign::realignForward(NB)--", aLo, aHi, bLo, bHi, _editDist->Left_Delta, _editDist->Left_Delta_Len, true, false); } } void NDalign::realignBackward(bool displayAlgorithm, bool displayAlign) { Match_Node_t match; match.Start = aend(); // Begin position in a match.Offset = bend(); // Begin position in b match.Len = 0; match.Next = 0; // Not used here int32 aLo=0, aHi=0; int32 bLo=0, bHi=0; char origScore[1024]; if (displayAlgorithm) fprintf(stderr, "NDalign::realignBackward()--\n"); pedOverlapType olapType = _editDist->Extend_Alignment(&match, // Initial exact match, relative to start of string _aStr, _aLen, _bStr, _bLen, aLo, aHi, // Output: Regions which the match extends bLo, bHi); aHi++; // Add one to the end point because Extend_Alignment returns the base-based coordinate. bHi++; // Is this a better overlap than what we have? if (((score() < _editDist->score())) || ((score() <= _editDist->score()) && (length() > ((aHi - aLo) + (bHi - bLo) + _editDist->Left_Delta_Len) / 2))) { if (displayAlign) { fprintf(stderr, "NDalign::realignBackward()--\n"); fprintf(stderr, "NDalign::realignBackward()-- Save better alignment\n"); fprintf(stderr, "NDalign::realignBackward()--\n"); sprintf(origScore, "NDalign::realignBackward()-- OLD length %u erate %f score %u (%d-%d %d-%d)\n", length(), erate(), score(), abgn(), aend(), bbgn(), bend()); } _bestResult.save(aLo, aHi, bLo, bHi, _editDist->score(), olapType, _editDist->Left_Delta_Len, _editDist->Left_Delta); display("NDalign::realignBackward()-- ", displayAlign); _bestResult.setErate(1.0 - (double)(_matches + _gapmatches) / (length() - _freegaps)); if (displayAlign) { fprintf(stderr, "%sNDalign::realignBackward()-- NEW length %u erate %f score %u (%d-%d %d-%d)\n", origScore, length(), erate(), score(), abgn(), aend(), bbgn(), bend()); } } else if (displayAlign) { fprintf(stderr, "NDalign::realignBackward()-- Alignment no better - OLD length %u erate %f score %u (%d-%d %d-%d)\n", length(), erate(), score(), abgn(), aend(), bbgn(), bend()); fprintf(stderr, "NDalign::realignBackward()-- Alignment no better - NEW length %u erate %f score %u (%d-%d %d-%d)\n", ((aHi - aLo) + (bHi - bLo) + _editDist->Left_Delta_Len) / 2, 0.0, _editDist->score(), aLo, aHi, bLo, bHi); //display("NDalign::realignBackward(NB)--", aLo, aHi, bLo, bHi, _editDist->Left_Delta, _editDist->Left_Delta_Len, true, false); } } void NDalign::display(char *prefix, int32 aLo, int32 aHi, int32 bLo, int32 bHi, int32 *delta, int32 deltaLen, bool displayIt, bool saveStats) { uint32 matches = 0; uint32 errors = 0; uint32 gapmatches = 0; uint32 freegaps = 0; if (_topDisplay == NULL) { _topDisplay = new char [2 * AS_MAX_READLEN + 1]; _botDisplay = new char [2 * AS_MAX_READLEN + 1]; _resDisplay = new char [2 * AS_MAX_READLEN + 1]; } char *a = astr() + aLo; int32 a_len = aHi - aLo; char *b = bstr() + bLo; int32 b_len = bHi - bLo; int32 top_len = 0; { int32 i = 0; int32 j = 0; for (int32 k = 0; k < deltaLen; k++) { for (int32 m = 1; m < abs (delta[k]); m++) { _topDisplay[top_len++] = a[i++]; j++; } if (delta[k] < 0) { _topDisplay[top_len++] = '-'; j++; } else { _topDisplay[top_len++] = a[i++]; } } while (i < a_len && j < b_len) { _topDisplay[top_len++] = a[i++]; j++; } _topDisplay[top_len] = 0; } int32 bot_len = 0; { int32 i = 0; int32 j = 0; for (int32 k = 0; k < deltaLen; k++) { for (int32 m = 1; m < abs (delta[k]); m++) { _botDisplay[bot_len++] = b[j++]; i++; } if (delta[k] > 0) { _botDisplay[bot_len++] = '-'; i++; } else { _botDisplay[bot_len++] = b[j++]; } } while (j < b_len && i < a_len) { _botDisplay[bot_len++] = b[j++]; i++; } _botDisplay[bot_len] = 0; } // Compute stats, build the result display. { for (int32 i=0; (i < top_len) || (i < bot_len); i++) { char T = _topDisplay[i]; char B = _botDisplay[i]; char t = tolower(T); char b = tolower(B); // A minor-allele match if the lower case matches, but upper case doesn't if ((t == b) && (T != B)) { _resDisplay[i] = '\''; gapmatches++; } // A free-gap if either is lowercase else if (((t == T) && (b == '-')) || // T is lowercase, B is a gap ((b == B) && (t == '-'))) { // B is lowercase, T is a gap _resDisplay[i] = '-'; freegaps++; } // A match if the originals match else if ((t == b) && (T == B)) { _resDisplay[i] = ' '; matches++; } // Otherwise, an error else { _resDisplay[i] = '^'; errors++; } } _resDisplay[ max(top_len, bot_len) ] = 0; } // Really display it? if (displayIt == true) { for (int32 i=0; (i < top_len) || (i < bot_len); i += DISPLAY_WIDTH) { fprintf(stderr, "%s%d\n", prefix, i); fprintf(stderr, "%sA: ", prefix); for (int32 j=0; (j < DISPLAY_WIDTH) && (i+j < top_len); j++) putc(_topDisplay[i+j], stderr); fprintf(stderr, "\n"); fprintf(stderr, "%sB: ", prefix); for (int32 j=0; (j < DISPLAY_WIDTH) && (i+j < bot_len); j++) putc(_botDisplay[i+j], stderr); fprintf(stderr, "\n"); fprintf(stderr, "%s ", prefix); for (int32 j=0; (j #include #include using namespace std; class NDalignResult { public: NDalignResult() { clear(); _deltaMax = 0; _delta = NULL; }; ~NDalignResult() { delete [] _delta; }; // Accessors double score(void) { return(_olapScore); }; int32 deltaLen(void) { return(_deltaLen); }; int32 *delta(void) { return(_delta); }; // Setters void clear(void) { _aLo = 0; _aHi = 0; _bLo = 0; _bHi = 0; _olapLen = 0; _olapScore = DBL_MIN; _olapErate = 0; _olapType = pedBothBranch; _deltaLen = 0; }; void save(int32 aLo, int32 aHi, int32 bLo, int32 bHi, int32 score, pedOverlapType type, int32 dl, int32 *d) { _aLo = aLo; _aHi = aHi; _bLo = bLo; _bHi = bHi; _olapLen = ((aHi - aLo) + (bHi - bLo) + dl) / 2; _olapScore = score; _olapErate = DBL_MAX; _olapType = type; _deltaLen = dl; if (_deltaMax < _deltaLen) { delete [] _delta; _deltaMax = _deltaLen + 4096; _delta = new int32 [_deltaMax]; } memcpy(_delta, d, sizeof(int32) * _deltaLen); }; void setErate(double olapErate) { _olapErate = olapErate; }; int32 _aLo; int32 _aHi; int32 _bLo; int32 _bHi; int32 _olapLen; double _olapScore; double _olapErate; pedOverlapType _olapType; int32 _deltaMax; int32 _deltaLen; int32 *_delta; }; class NDalign { public: NDalign(pedAlignType alignType, double maxErate, int32 merSize); ~NDalign(); void initialize(uint32 aID, char *aStr, int32 aLen, int32 aLo, int32 aHi, uint32 bID, char *bStr, int32 bLen, int32 bLo, int32 bHi, bool bFlipped); // Algorithm bool findMinMaxDiagonal(int32 minLength, uint32 AbgnAdj=0, uint32 AendAdj=0, uint32 BbgnAdj=0, uint32 BendAdj=0); bool findSeeds(bool dupIgnore); bool findHits(void); bool chainHits(void); bool makeNullHit(void); bool processHits(void); // Diagnostic bool scanDeltaForBadness(bool verbose=false, bool showAlign=false); void realignForward(bool verbose=false, bool displayAlign=false); void realignBackward(bool verbose=false, bool displayAlign=false); void display(char *prefix, int32 aLo, int32 aHi, int32 bLo, int32 bHi, int32 *delta, int32 deltaLen, bool displayIt = true, bool saveStats = false); void display(char *prefix, bool displayIt = true); // Result reporting char *astr(void) { return(_aStr); }; char *bstr(void) { return(_bStr); }; int32 abgn(void) { return(_bestResult._aLo); }; int32 aend(void) { return(_bestResult._aHi); }; int32 bbgn(void) { return((_bFlipped == false) ? (_bestResult._bLo) : (_bestResult._bLo)); }; int32 bend(void) { return((_bFlipped == false) ? (_bestResult._bHi) : (_bestResult._bHi)); }; int32 ahg5(void) { return( _bestResult._aLo); }; int32 ahg3(void) { return(_aLen - _bestResult._aHi); }; int32 bhg5(void) { return((_bFlipped == false) ? ( _bestResult._bLo) : (_bLen - _bestResult._bLo)); }; int32 bhg3(void) { return((_bFlipped == false) ? (_bLen - _bestResult._bHi) : ( _bestResult._bHi)); }; int32 length(void) { return(_bestResult._olapLen); }; double erate(void) { return(_bestResult._olapErate); }; int32 score(void) { return(_bestResult._olapScore); }; pedOverlapType type(void) { return(_bestResult._olapType); }; int32 deltaLen(void) { return(_bestResult.deltaLen()); }; int32 *delta(void) { return(_bestResult.delta()); }; private: class exactMatch { public: exactMatch(int32 a, int32 b, int32 l) { aBgn = a; bBgn = b; tLen = l; }; int32 aBgn; // Signed to allow for easy compute of diagonal int32 bBgn; int32 tLen; bool operator<(exactMatch const that) const { if (tLen == that.tLen) return(aBgn < that.aBgn); return(tLen > that.tLen); }; }; // Parameters of the alignment pedAlignType _alignType; double _maxErate; int32 _merSizeInitial; // From the parameters. uint32 _aID; char *_aStr; int32 _aLen; int32 _aLoOrig; int32 _aHiOrig; uint32 _bID; char *_bStr; int32 _bLen; int32 _bLoOrig; int32 _bHiOrig; // If flipped, we need to save the reverse complement somewhere bool _bFlipped; uint32 _bRevMax; char *_bRev; // Computed stuff, for each alignment. NDalgorithm *_editDist; int32 _minDiag; int32 _maxDiag; int32 _merSize; map _aMap; // Signed, to allow for easy compute of diagonal map _bMap; vector _rawhits; vector _hits; uint32 _hitr; // The result. NDalignResult _bestResult; // For displaying and stats. char *_topDisplay; char *_botDisplay; char *_resDisplay; uint32 _matches; uint32 _errors; uint32 _gapmatches; uint32 _freegaps; // Constants uint64 acgtToBit[256]; uint64 acgtToVal[256]; uint64 merMask[33]; void fastFindMersA(bool dupIgnore); void fastFindMersB(bool dupIgnore); }; #endif // NDALIGN_H canu-1.3/src/utgcns/libNNalign/000077500000000000000000000000001272335422300164155ustar00rootroot00000000000000canu-1.3/src/utgcns/libNNalign/NNalgorithm.C000066400000000000000000000273231272335422300207520ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/alignment/AS_ALN_bruteforcedp.C * src/alignment/brute-force-dp.C * * Modifications by: * * Brian P. Walenz from 2014-DEC-23 to 2015-AUG-10 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "aligners.H" #include "gkStore.H" #include "AS_UTL_reverseComplement.H" #undef DEBUG #define STOP 0 // IMPORTANT; default is STOP #define MATCH 1 #define GAPA 2 #define GAPB 3 // 3, -2, -2 original // 5, -4, -3 suggested // 3, -6, -4 got around one problem, but made many more // #define MATCHSCORE 3 #define GAPSCORE -6 #define MISMATCHSCORE -4 #define SLOP 10 // Scores in the dynamic programming matrix are unsigned ints, currently 30 bits wide. // // The smallest score is 0. // The zero score is 2^29 (512 million) // The largest score is 2^30-1 (1 billion). // // Edges that we don't want to end up at are initialized to DP_NEGT (2^28, 256 million) which will // take an awful lot of alignment to make it decent, or to underflow. // // Edges that we want to end up at are set to DP_ZERO (2^29, 512 million). #define DP_NEGT (1 << 28) // A very negative value #define DP_ZERO (1 << 29) // Zero class dpActions { public: dpActions() { actions = new uint64 * [2 * AS_MAX_READLEN]; memset(actions, 0, 2 * AS_MAX_READLEN * sizeof(uint64 *)); }; ~dpActions() { for (uint32 i=0; i<2 * AS_MAX_READLEN; i++) if (actions[i] != NULL) delete [] actions[i]; delete [] actions; }; // Pointer to array of packed bits. uint64 **actions; uint32 *actionLength; void allocate(uint32 i) { if (actions[i] != NULL) return; uint32 len = 2 * AS_MAX_READLEN * 2 / 64 + 1; actions[i] = new uint64 [len]; memset(actions[i], 0, len * sizeof(uint64)); // IMPORTANT, init to STOP }; uint64 get(uint32 i, uint32 j) { allocate(i); uint32 addr = ((j >> 5) & 0x07ffffff); uint32 bits = ((j ) & 0x0000001f) << 1; uint64 v = (actions[i][addr] >> bits) & 0x00000003llu; //fprintf(stderr, "GET %u %u %lu (%u %u)\n", i, j, v, addr, bits); return(v); }; void set(uint32 i, uint32 j, uint64 v) { allocate(i); assert(v < 4); uint32 addr = ((j >> 5) & 0x07ffffff); uint32 bits = ((j ) & 0x0000001f) << 1; //fprintf(stderr, "SET %u %u to %lu (%u %u)\n", i, j, v, addr, bits); actions[i][addr] &= ~(0x00000003llu << bits); actions[i][addr] |= (v << bits); }; }; // // ahang, bhang represent any sequence to EXCLUDE from the alignmet. There is a little bit of slop // in this exclusion. // // Setting both to the length of the sequence will try to find an alignment over all bases. The // highest scoring alignment is returned, which is likely not an alignment that uses all bases -- // the only requirement (assuming endToEnd is set) is that the alignment reaches the end of one // sequence. // void alignLinker(char *alignA, char *alignB, char *stringA, char *stringB, alignLinker_s *a, int endToEnd, int allowNs, int ahang, int bhang) { int32 lenA = strlen(stringA); int32 lenB = strlen(stringB); memset(a, 0, sizeof(alignLinker_s)); dpActions ACT; if ((lenA > AS_MAX_READLEN) || (lenB > AS_MAX_READLEN)) { fprintf(stderr, "alignLinker()-- Reads too long. %d or %d > %d\n", lenA, lenB, AS_MAX_READLEN); return; } uint32 *lastCol = new uint32 [AS_MAX_READLEN * 2]; uint32 *thisCol = new uint32 [AS_MAX_READLEN * 2]; uint32 *iFinal = new uint32 [AS_MAX_READLEN * 2]; uint32 *jFinal = new uint32 [AS_MAX_READLEN * 2]; //memset(lastCol, 0, sizeof(uint32) * AS_MAX_READLEN * 2); //memset(thisCol, 0, sizeof(uint32) * AS_MAX_READLEN * 2); //memset(iFinal, 0, sizeof(uint32) * AS_MAX_READLEN * 2); //memset(jFinal, 0, sizeof(uint32) * AS_MAX_READLEN * 2); // Definition of the box we want to do dynamic programming in. int32 ibgn = 1; int32 iend = lenA; int32 jbgn = 1; int32 jend = lenB; // Set the edges. for (int32 i=ibgn-1; i<=iend+1; i++) { iFinal[i] = DP_ZERO; } for (int32 j=jbgn-1; j<=jend+1; j++) { lastCol[j] = DP_ZERO; thisCol[j] = DP_ZERO; jFinal[j] = DP_ZERO; } // Catch an invalid use case if ((endToEnd == true) && ((ahang != 0) || (bhang != 0))) { fprintf(stderr, "Invalid code path. I shouldn't be here. Please report to atg@jcvi.org, thanks.\n"); assert(0); } int32 scoreMax = 0; int32 endI=0, curI=0; int32 endJ=0, curJ=0; #ifdef DEBUG fprintf(stderr, "%d,%d - %d,%d -- ahang,bhang %d,%d alen,blen %d,%d\n", ibgn, jbgn, iend, jend, ahang, bhang, lenA, lenB); #endif for (int32 i=ibgn; i<=iend; i++) { for (int32 j=jbgn; j<=jend; j++) { // Pick the max of these // i-1 -> lastCol // i -> thisCol uint32 ul = lastCol[j-1] + ((stringA[i-1] == stringB[j-1]) ? MATCHSCORE : MISMATCHSCORE); uint32 lf = lastCol[j ] + GAPSCORE; // Gap in B uint32 up = thisCol[j-1] + GAPSCORE; // Gap in A // When computing unitig consensus, the N letter is used to indicate a gap. This might be a // match, or it might be a mismatch. We just don't know. Don't count it as anything. This // should be extended to allow ambiguitity codes -- but that then needs base calling support. // if (allowNs) if ((stringA[i-1] == 'N') || (stringA[i-1] == 'n') || (stringB[j-1] == 'N') || (stringB[j-1] == 'n')) { ul = lastCol[j-1] + MATCHSCORE; } // For unitig consensus, if the consensus sequence (stringA) is lowercase, count it as a // match, otherwise ignore the gap it induces in stringB. // #warning stop complaing that this is ambiguous, please. if (stringA[i-1] == 'a') if (stringB[j-1] == 'A') ul = lastCol[j-1] + MATCHSCORE; else { ul = lastCol[j-1]; lf = lastCol[j]; } if (stringA[i-1] == 'c') if (stringB[j-1] == 'C') ul = lastCol[j-1] + MATCHSCORE; else { ul = lastCol[j-1]; lf = lastCol[j]; } if (stringA[i-1] == 'g') if (stringB[j-1] == 'G') ul = lastCol[j-1] + MATCHSCORE; else { ul = lastCol[j-1]; lf = lastCol[j]; } if (stringA[i-1] == 't') if (stringB[j-1] == 'T') ul = lastCol[j-1] + MATCHSCORE; else { ul = lastCol[j-1]; lf = lastCol[j]; } if (endToEnd) { // Set score to the smallest value possible; we will then ALWAYS pick an action below. // thisCol[j] = 0; ACT.set(i, j, MATCH); assert(ACT.get(i, j) == MATCH); } else { // (i,j) is the beginning of a subsequence, our default behavior. If we're looking for // local alignments, make the alignment stop here. // thisCol[j] = DP_ZERO; ACT.set(i, j, STOP); assert(ACT.get(i, j) == STOP); } if (thisCol[j] < ul) { thisCol[j] = ul; ACT.set(i, j, MATCH); assert(ACT.get(i, j) == MATCH); } if (thisCol[j] < lf) { thisCol[j] = lf; ACT.set(i, j, GAPB); assert(ACT.get(i, j) == GAPB); } if (thisCol[j] < up) { thisCol[j] = up; ACT.set(i, j, GAPA); assert(ACT.get(i, j) == GAPA); } if (scoreMax < thisCol[j]) { scoreMax = thisCol[j]; endI = curI = i; endJ = curJ = j; } if (j == jend) jFinal[i] = thisCol[j]; if (i == iend) iFinal[j] = thisCol[j]; // Not really needed (just a copy of thisCol), but simplifies later code } // Over j // Swap the columns uint32 *tc = thisCol; thisCol = lastCol; lastCol = tc; } // Over i // If we're not looking for local alignments, scan the end points for the best value. If we are // looking for local alignments, we've already found and remembered the best end point. // // lastCol - [iend][] // jFinal - [][jend] // if (endToEnd) { //fprintf(stderr, "ENDTOEND curI %u curJ %u\n", curI, curJ); scoreMax = 0; endI = curI = 0; endJ = curJ = 0; for (int32 i=ibgn, j=jend; i<=iend; i++) { if (scoreMax < jFinal[i]) { scoreMax = jFinal[i]; endI = curI = i; endJ = curJ = j; //fprintf(stderr, "IscoreMax = %d at %d,%d\n", scoreMax - DP_ZERO, i, j); } } for (int32 j=jbgn, i=iend; j<=jend; j++) { if (scoreMax < iFinal[j]) { scoreMax = iFinal[j]; endI = curI = i; endJ = curJ = j; //fprintf(stderr, "JscoreMax = %d at %d,%d\n", scoreMax - DP_ZERO, i, j); } } // Not sure what this is for. //M[endI][endJ].score = 0; } //fprintf(stderr, "FINAL curI %u curJ %u\n", curI, curJ); int32 alignLen = 0; int32 nGapA = 0; int32 nGapB = 0; int32 nMatch = 0; int32 nMismatch = 0; int32 terminate = 0; while (terminate == 0) { uint32 act = ACT.get(curI, curJ); //fprintf(stderr, "ACTION %2u curI %u curJ %u\n", act, curI, curJ); switch (act) { case STOP: terminate = 1; break; case MATCH: alignA[alignLen] = stringA[curI-1]; alignB[alignLen] = stringB[curJ-1]; if (alignA[alignLen] == alignB[alignLen]) { alignA[alignLen] = tolower(alignA[alignLen]); alignB[alignLen] = tolower(alignB[alignLen]); nMatch++; } else { alignA[alignLen] = toupper(alignA[alignLen]); alignB[alignLen] = toupper(alignB[alignLen]); nMismatch++; } curI--; curJ--; alignLen++; break; case GAPA: alignA[alignLen] = '-'; alignB[alignLen] = stringB[curJ-1]; curJ--; alignLen++; nGapA++; break; case GAPB: alignA[alignLen] = stringA[curI-1]; alignB[alignLen] = '-'; curI--; alignLen++; nGapB++; break; } } alignA[alignLen] = 0; alignB[alignLen] = 0; reverse(alignA, alignB, alignLen); a->matches = nMatch; a->alignLen = alignLen; a->begI = curI; a->begJ = curJ; a->endI = endI; a->endJ = endJ; a->lenA = lenA; a->lenB = lenB; a->pIdentity = (double)(nMatch) / (double)(nGapA + nGapB + nMatch + nMismatch); a->pCoverageA = (double)(a->endI - a->begI) / (double)(lenA); a->pCoverageB = (double)(a->endJ - a->begJ) / (double)(lenB); delete [] lastCol; delete [] thisCol; delete [] iFinal; delete [] jFinal; } canu-1.3/src/utgcns/libNNalign/NNalign.C000066400000000000000000000144441272335422300200560ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/alignment/AS_ALN_forcns.C * src/alignment/alignment-drivers.C * * Modifications by: * * Brian P. Walenz from 2014-DEC-23 to 2015-AUG-10 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "aligners.H" #include "gkStore.H" // For AS_MAX_READLEN #include "AS_UTL_reverseComplement.H" class Optimal_Overlap_Data { public: char h_alignA[AS_MAX_READLEN + AS_MAX_READLEN + 2]; char h_alignB[AS_MAX_READLEN + AS_MAX_READLEN + 2]; int h_trace[AS_MAX_READLEN + AS_MAX_READLEN + 2]; ALNoverlap o; }; ALNoverlap * Optimal_Overlap_AS_forCNS(char *a, char *b, int begUNUSED, int endUNUSED, int ahang, int bhang, int opposite, double erate, double thresh, int minlen, CompareOptions what) { static Optimal_Overlap_Data *ood = NULL; if (ood == NULL) ood = new Optimal_Overlap_Data; memset(ood->h_alignA, 0, sizeof(char) * (AS_MAX_READLEN + AS_MAX_READLEN + 2)); memset(ood->h_alignB, 0, sizeof(char) * (AS_MAX_READLEN + AS_MAX_READLEN + 2)); memset(ood->h_trace, 0, sizeof(int) * (AS_MAX_READLEN + AS_MAX_READLEN + 2)); alignLinker_s al; //if (VERBOSE_MULTIALIGN_OUTPUT >= 3) // fprintf(stderr, "Optimal_Overlap_AS_forCNS()-- Begins\n"); #if 0 if (erate > AS_MAX_ERROR_RATE) { //fprintf(stderr, "Optimal_Overlap_AS_forCNS()-- erate=%f >= AS_MAX_ERROR_RATE=%f, reset to max\n", erate, (double)AS_MAX_ERROR_RATE); erate = AS_MAX_ERROR_RATE; } assert((0.0 <= erate) && (erate <= AS_MAX_ERROR_RATE)); #endif if (opposite) reverseComplementSequence(b, strlen(b)); //if (VERBOSE_MULTIALIGN_OUTPUT >= 3) { // fprintf(stderr, "ALIGN %s\n", a); // fprintf(stderr, "ALIGN %s\n", b); //} alignLinker(ood->h_alignA, ood->h_alignB, a, b, &al, true, // Looking for global end-to-end alignments false, // Count matches to N as matches ahang, bhang); if (al.alignLen == 0) { return NULL; } //if (VERBOSE_MULTIALIGN_OUTPUT >= 3) { // fprintf(stderr, "ALIGN %d %d-%d %d-%d opposite=%d\n", al.alignLen, al.begI, al.endI, al.begJ, al.endJ, opposite); // fprintf(stderr, "ALIGN '%s'\n", ood->h_alignA); // fprintf(stderr, "ALIGN '%s'\n", ood->h_alignB); //} if (opposite) { reverseComplementSequence(b, strlen(b)); reverseComplementSequence(ood->h_alignA, al.alignLen); reverseComplementSequence(ood->h_alignB, al.alignLen); int x = al.begJ; al.begJ = al.lenB - al.endJ; al.endJ = al.lenB - x; } // We don't expect partial overlaps here. At least one fragment // must have an alignment to the very start. // // ECR depends on this return value; it is allowed to fail // when building a new unitig multialign. For example: // // <----------------------- // ------> // // When ECR tries to extend the second fragment, it checks that // the extended fragment overlaps the next contig. It does not // check that the extended bits agree with the first fragment, // leaving that up to "does the unitig rebuild". // if ((al.begJ != 0) && (al.begI != 0)) return(NULL); ood->o.begpos = (al.begI > 0) ? (al.begI) : -(al.begJ); ood->o.endpos = (al.lenB - al.endJ > 0) ? (al.lenB - al.endJ) : -(al.lenA - al.endI); ood->o.length = al.alignLen; ood->o.diffs = 0; ood->o.comp = opposite; ood->o.trace = ood->h_trace; { int x=0; int tp = 0; int ap = al.begI; int bp = al.begJ; for (x=0; xh_alignA[x] == '-') { ood->h_trace[tp++] = -(ap + 1); ap--; } if (ood->h_alignB[x] == '-') { ood->h_trace[tp++] = bp + 1; bp--; } // Count the differences. // // But allow N's and lowercase as matches. If either letter is N, then the other letter is // NOT N (if both letters were N, both would be lowercase n, representing a match). This // just subtracts out the diff we added in above. // bool diff = false; bool ignore = false; if (toupper(ood->h_alignA[x]) != toupper(ood->h_alignB[x])) diff = true; if ((ood->h_alignA[x] == 'N') || (ood->h_alignA[x] == 'n') || (ood->h_alignB[x] == 'N') || (ood->h_alignB[x] == 'n')) ignore = true; if (islower(ood->h_alignA[x]) && (ood->h_alignB[x] == '-')) ignore = true; if ((diff == true) && (ignore == false)) ood->o.diffs++; bp++; ap++; } ood->h_trace[tp] = 0; //if (VERBOSE_MULTIALIGN_OUTPUT >= 4) { // fprintf(stderr, "trace"); // for (x=0; xh_trace[x]); // fprintf(stderr, "\n"); // fprintf(stderr, "A: %4d-%4d %4d %s\n", al.begI, al.endI, al.lenA, ood->h_alignA); // fprintf(stderr, "B: %4d-%4d %4d %s\n", al.begJ, al.endJ, al.lenB, ood->h_alignB); //} } //if (VERBOSE_MULTIALIGN_OUTPUT >= 3) { // fprintf(stderr, "ERATE: diffs=%d / length=%d = %f\n", ood->o.diffs, ood->o.length, (double)ood->o.diffs / ood->o.length); // fprintf(stderr, "Optimal_Overlap_AS_forCNS()-- Ends\n"); //} if ((double)ood->o.diffs / ood->o.length <= erate) return(&ood->o); return(NULL); } canu-1.3/src/utgcns/libNNalign/NNalign.H000066400000000000000000000034261272335422300200610ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/alignment/AS_ALN_aligners.H * src/alignment/aligners.H * * Modifications by: * * Brian P. Walenz from 2014-DEC-23 to 2015-AUG-10 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-11 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef ALIGNERS_H #define ALIGNERS_H #include "AS_global.H" // // BPW's brute-force dynamic programming // typedef struct { int32 matches; int32 alignLen; int32 begI, begJ; int32 endI, endJ; int32 lenA, lenB; double pIdentity; double pCoverageA; double pCoverageB; } alignLinker_s; void alignLinker(char *alignA, char *alignB, char *stringA, char *stringB, alignLinker_s *a, int endToEnd, int allowNs, int ahang, int bhang); #endif canu-1.3/src/utgcns/libcns/000077500000000000000000000000001272335422300156525ustar00rootroot00000000000000canu-1.3/src/utgcns/libcns/NOTES000066400000000000000000000011521272335422300164640ustar00rootroot00000000000000 BeadLinks (prev/this/next) M=max #0 <--> 0/0/0 <--> 0/0/0 <--> #1 <\ M/1/1 <--> 1/1/1 <--> #2 <\\> 1/2/M /> 3/2/2 <--> #3 \> 2/3/2 0; pp++) { _bases[pp] = inv[ seq[ii] ]; _quals[pp] = qlt[ii]; } _bases[_length] = 0; // NUL terminate the strings so we can use them in aligners. _quals[_length] = 0; // Not actually a string, the 0 is just another QV=0 entry. }; void abAbacus::addRead(gkStore *gkpStore, uint32 readID, uint32 askip, uint32 bskip, bool complemented, map *inPackageRead, map *inPackageReadData) { // Grab the read. If there is no package, load the read from the store. Otherwise, load the // read from the package. This REQUIRES that the package be in-sync with the unitig. We fail // otherwise. Hey, it's used for debugging only... gkRead *read = NULL; gkReadData *readData = NULL; if (inPackageRead == NULL) { read = gkpStore->gkStore_getRead(readID); readData = new gkReadData; gkpStore->gkStore_loadReadData(read, readData); } else { read = (*inPackageRead)[readID]; readData = (*inPackageReadData)[readID]; } assert(read != NULL); assert(readData != NULL); // Grab seq/qlt from the read, offset to the proper begin and length. uint32 seqLen = read->gkRead_sequenceLength() - askip - bskip; char *seq = readData->gkReadData_getSequence() + ((complemented == false) ? askip : bskip); char *qlt = readData->gkReadData_getQualities() + ((complemented == false) ? askip : bskip); // Tell abacus about it. We could pre-allocate _sequences (in the constructor) but this is // relatively painless and makes life easier outside here. increaseArray(_sequences, _sequencesLen, _sequencesMax, 1); _sequences[_sequencesLen++] = new abSequence(readID, seqLen, seq, qlt, complemented); delete readData; } canu-1.3/src/utgcns/libcns/abAbacus-appendBases.C000066400000000000000000000066371272335422300217160ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz beginning on 2015-NOV-23 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "abAbacus.H" // Just appends bases to the frankenstein sequence. // Sets read abacus pointers assuming gapless alignments. void abAbacus::appendBases(uint32 bid, uint32 bgn, uint32 end) { uint32 alen = numberOfColumns(); // Set up the b read abSequence *bseq = getSequence(bid); uint32 blen = bseq->length(); // Find the first and last columns the read 'aligns' to. If the read isn't contained, the last // column is the last column in the frankenstein. We'll append new columns after this one. // // Space-based, so the end element is the (n-1)th element. // abColumn *fc = getColumn(bgn); abColumn *lc = (end <= alen) ? getColumn(end-1) : getLastColumn(); uint16 fl = UINT16_MAX; uint16 ll = UINT16_MAX; // To get positions in output, we need to have both the first and last beads // placed in the multialign. The first bead is always aligned, but the last bead // is aligned only if it is contained. fl = fc->alignBead(UINT16_MAX, bseq->getBase(0), bseq->getQual(0)); if (end <= alen) ll = lc->alignBead(UINT16_MAX, bseq->getBase(blen-1), bseq->getQual(blen-1)); // If not contained, push on bases, and update the consensus base. This is all _very_ rough. // The unitig-supplied coordinates aren't guaranteed to contain 'blen' bases. We make the // obvious guess as to where the frankenstein ends in the read, and push on bases from there till // the end. // // The obvious guess is to add exactly enough bases to make the new frankenstein end at // the expected length. // // alen // v blen // -------------------- v // -------------------------- // ^ ^ ^ // bgn bpos end !! end-bgn != blen !! // else for (uint32 bpos=blen - (end - alen); bposinsertAtEnd(lc, UINT16_MAX, bseq->getBase(bpos), bseq->getQual(bpos)); lc = nc; //baseCallMajority(lc); } // Now set the first/last links. beadID f(fc, fl); beadID l(lc, ll); readTofBead[bid] = f; fbeadToRead[f] = bid; readTolBead[bid] = l; lbeadToRead[l] = bid; // If we did this correctly, then the first/last column indices should agree with the read placement. //assert(bgn == getBead(bfrag->firstBead())->colIdx().get()); //assert(end-1 == getBead(bfrag->lastBead())->colIdx().get()); } canu-1.3/src/utgcns/libcns/abAbacus-applyAlignment.C000066400000000000000000000555051272335422300224530ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/ApplyAlignment.C * src/AS_CNS/ApplyAlignment.c * src/AS_CNS/MultiAlignment_CNS.c * src/utgcns/libcns/ApplyAlignment.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-MAR-30 to 2008-FEB-13 * are Copyright 2005-2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Gennady Denisov from 2005-MAY-09 to 2008-JUN-06 * are Copyright 2005-2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2005-SEP-29 to 2006-OCT-03 * are Copyright 2005-2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-FEB-27 to 2009-MAY-14 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-JUL-28 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-13 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "abAbacus.H" #undef DEBUG_INFER #undef DEBUG_ABACUS_ALIGN // Primary debug output, shows progress of the algorithm void abColumn::allocateInitialBeads(void) { // Allocate beads. We'll need no more than the max of either the prev or the next. Any read that we // interrupt gets a new gap bead. Any read that has just ended gets nothing. And, +1 for the read // we might be adding to the multialign. uint32 pmax = (_prevColumn != NULL) ? (_prevColumn->depth() + 1) : (4); uint32 nmax = (_nextColumn != NULL) ? (_nextColumn->depth() + 1) : (4); _beadsMax = MAX(pmax, nmax); _beadsLen = 0; _beads = new abBead [_beadsMax]; for (uint32 ii=0; ii<_beadsMax; ii++) // Probably done by the constructor. _beads[ii].clear(); } void abColumn::inferPrevNextBeadPointers(void) { #ifdef DEBUG_INFER fprintf(stderr, "infer between columns %d and %d\n", _prevColumn->position(), _nextColumn->position()); #endif // Scan the next/prev lists, adding a gap bead for any read that is spanned. // // prev/this/next prev/this/next // 0 0 0 <-----> 0 0 0 // A spanned read. // 1 1 MAX /--> 2 1 1 // The first read ended // 2 2 1 <-/ /-> 3 2 2 // Now columns are different // 3 3 3 <--/ assert(_prevColumn != NULL); assert(_nextColumn != NULL); // At startup, the links in this column are all invalid, and the base hasn't been added. assert(_prevColumn->_beadsLen <= _prevColumn->_beadsMax); assert( _beadsLen == 0); assert(_nextColumn->_beadsLen <= _nextColumn->_beadsMax); uint32 pmax = (_prevColumn == NULL) ? 0 : _prevColumn->_beadsLen; uint32 nmax = (_nextColumn == NULL) ? 0 : _nextColumn->_beadsLen; uint32 ppos = 0; // Position in the prev's nextOffset list uint32 tpos = 0; // Position in our offset list uint32 npos = 0; // Position in the next's prevOffset list // Add gaps and link into any existing reads. while (1) { // Find reads to work on. while ((ppos < pmax) && // If the previous position is less than the depth, and (_prevColumn->_beads[ppos].nextOffset() == UINT16_MAX)) // the previous pointer is not valid, ppos++; // move ahead to the next read. while ((npos < nmax) && (_nextColumn->_beads[npos].prevOffset() == UINT16_MAX)) npos++; // If either column ran out of entries, we're done. No more reads will span this new column. if ((ppos >= _prevColumn->depth()) || (npos >= _nextColumn->depth())) break; // The columns must agree. uint16 no = _prevColumn->_beads[ppos].nextOffset(); uint16 po = _nextColumn->_beads[npos].prevOffset(); if ((no != npos) || (po != ppos)) { fprintf(stderr, "ERROR: link mismatch.\n"); fprintf(stderr, "ERROR: prev %d %d -- next %d %d\n", _prevColumn->_beads[ppos].prevOffset(), _prevColumn->_beads[ppos].nextOffset(), _nextColumn->_beads[npos].prevOffset(), _nextColumn->_beads[npos].nextOffset()); } assert(no == npos); // The 'next-bead' that the prev is pointing to must be npos. assert(po == ppos); // The 'prev-bead' that the next is pointing to must be ppos. // Reset the prev/next offsets to point to us. _prevColumn->_beads[ppos]._nextOffset = tpos; _nextColumn->_beads[npos]._prevOffset = tpos; // Set our offsets to point to them. _beads[tpos]._prevOffset = po; _beads[tpos]._nextOffset = no; _beadsLen = tpos + 1; // Needed for showLink() #ifdef DEBUG_INFER fprintf(stderr, "inferPrevNextBeadPointers()\n"); showLinks(ppos, tpos, npos); #endif // And move forward to the next read in all columns. ppos++; assert(ppos <= _prevColumn->_beadsLen); tpos++; assert(tpos <= _beadsMax); npos++; assert(npos <= _nextColumn->_beadsLen); } // Set the new length of the bead list. assert(_beadsLen == tpos); _beadsLen = tpos; #ifdef DEBUG_INFER fprintf(stderr, "inferPrevNextBeadPointers() final\n"); showLinks(); #endif } // Add a column, seeded with some base/qual assignment. The assignment cannot be a gap (because // all other reads in this column are set to gap). // // Returns the bead index for the read that was added. // // Can insert the new column before or after some specified column. In both cases, we need to // know the read index for the read in that column that we're adding a base to. // // Common use cases are to: // add a new column for a base aligned to a gap in the multialign // - all existing reads get a gap // - this read is assigned a base // // add a new column before/after the multialign // - this read is the only base in this column // // In both cases, we need to have and return linking information to the existing read. // // One odd ball case occurs when the read isn't in the muiltialign at all; when we're adding // the very first base. This is handled by setting the link to max. // // Insert a new empty column, seeded with only the base/qual supplied, to the start of the multialign. // This grows the multialign to add new bases to the start: // [original-multialign] // 1[original-multialign] // 12[original-multialign] // 123[original-multialign] // 1234[original-multialign] // uint16 abColumn::insertAtBegin(abColumn *first, uint16 prevLink, char base, uint8 qual) { // The base CAN NOT be a gap - the new column would then be entirely a gap column, with no base. assert(base != '-'); assert(base != 0); _columnPosition = INT32_MAX; _call = base; _qual = qual; _prevColumn = first->_prevColumn; _nextColumn = first; first->_prevColumn = this; if (_prevColumn) _prevColumn->_nextColumn = this; allocateInitialBeads(); _beads[0]._unused = 0; _beads[0]._isRead = 1; _beads[0]._isUnitig = 0; _beads[0]._base = base; _beads[0]._qual = qual; _beads[0]._prevOffset = prevLink; _beads[0]._nextOffset = UINT16_MAX; // No next offset for the next base. _beadsLen++; assert(_beadsLen <= _beadsMax); if (_prevColumn) _prevColumn->_beads[prevLink]._nextOffset = 0; #ifdef BASECOUNT baseCountIncr(base); #endif // With only the one read, we don't need to do any base calling here. #ifdef DEBUG_ABACUS_ALIGN fprintf(stderr, "insertAtBegin()-- column prev=%p next=%p link prev=%d this=%d next=%d\n", _prevColumn, this, _nextColumn, _beads[0]._prevOffset, _beads[0]._nextOffset); #endif return(0); } // Insert a new empty column, seeded with only the base/qual supplied, to the end of the multialign. // The growth is simpler than insertAtBegin() because the end is always NULL. // [original-multialign] // [original-multialign]7 // [original-multialign]78 // [original-multialign]789 // uint16 abColumn::insertAtEnd(abColumn *prev, uint16 prevLink, char base, uint8 qual) { assert(base != '-'); // The base CAN NOT be a gap - the new column would then be entirely a gap column, with no base. assert(base != 0); _columnPosition = INT32_MAX; _call = base; _qual = qual; _prevColumn = prev; _nextColumn = NULL; if (prev == NULL) assert(prevLink == UINT16_MAX); if (prev != NULL) assert(prevLink != UINT16_MAX); if (prev) prev->_nextColumn = this; allocateInitialBeads(); _beads[0]._unused = 0; _beads[0]._isRead = 1; _beads[0]._isUnitig = 0; _beads[0]._base = base; _beads[0]._qual = qual; _beads[0]._prevOffset = prevLink; _beads[0]._nextOffset = UINT16_MAX; _beadsLen++; assert(_beadsLen <= _beadsMax); if (prev) prev->_beads[prevLink]._nextOffset = 0; #ifdef BASECOUNT baseCountIncr(base); #endif // With only the one read, we don't need to do any base calling here. #ifdef DEBUG_ABACUS_ALIGN fprintf(stderr, "insertAtEnd()-- column prev=%p this=%p next=%p link prev=%d next=%d\n", _prevColumn, this, _nextColumn, _beads[0]._prevOffset, _beads[0]._nextOffset); #endif return(0); } // Insert a column in the middle of the multialign, after some column. uint16 abColumn::insertAfter(abColumn *prev, // Add new column after 'prev' uint16 prevLink, // The bead for this read in 'prev' is at 'prevLink'. char base, uint8 qual) { // The base CAN NOT be a gap - the new column would then be entirely a gap column, with no base. assert(base != '-'); assert(base != 0); // Make sure that we're actually in the middle of consensus. assert(prev != NULL); assert(prev->_nextColumn != NULL); // Link in the new column (us!) _prevColumn = prev; _nextColumn = prev->_nextColumn; _prevColumn->_nextColumn = this; _nextColumn->_prevColumn = this; // Allocate space for beads in this column (based on _prevColumn and _nextColumn) allocateInitialBeads(); // Add gaps for the existing reads. This is quite complicated, so stashed away in a closet where we won't see it. inferPrevNextBeadPointers(); // Set up the new consensus base. _columnPosition = INT32_MAX; _call = base; _qual = qual; // Then add the base for this read. allocateInitialBeads() guarantees space for at least one // more bead, infer() filled in the rest. uint16 tpos = _beadsLen++; assert(_beadsLen <= _beadsMax); _beads[tpos]._unused = 0; _beads[tpos]._isRead = 1; _beads[tpos]._isUnitig = 0; _beads[tpos]._base = base; _beads[tpos]._qual = qual; _beads[tpos]._prevOffset = prevLink; _beads[tpos]._nextOffset = UINT16_MAX; // Don't forget to update the link to us! (I forgot.) if (prevLink != UINT16_MAX) _prevColumn->_beads[prevLink]._nextOffset = tpos; #ifdef BASECOUNT baseCountIncr(base); #endif baseCall(false); // We need to recall the base, using the majority vote. #ifdef DEBUG_ABACUS_ALIGN fprintf(stderr, "insertAfter()-- column prev=%d this=%p next=%d link %d prev=%d next=%d\n", _prevColumn->position(), this, _nextColumn->position(), tpos, _beads[tpos]._prevOffset, _beads[tpos]._nextOffset); #endif #ifdef DEBUG_INFER showLinks(); #endif #ifdef CHECK_LINKS checkLinks(); #endif return(tpos); } uint16 abColumn::alignBead(uint16 prevIndex, char base, uint8 qual) { // First, make sure the column has enough space for the new read. increaseArray(_beads, _beadsLen, _beadsMax, 1); // Set up the new bead. uint16 tpos = _beadsLen++; assert(_beadsLen <= _beadsMax); _beads[tpos].initialize(base, qual, prevIndex, UINT16_MAX); // Link to the previous if ((_prevColumn) && (prevIndex != UINT16_MAX)) { assert(prevIndex < _prevColumn->_beadsLen); assert(prevIndex < _prevColumn->_beadsMax); _prevColumn->_beads[prevIndex]._nextOffset = tpos; } // increment the base count too! #ifdef BASECOUNT baseCountIncr(base); #endif baseCall(false); // We need to recall the base, using the majority vote. #ifdef CHECK_LINKS checkLinks(); #endif // Return the index of the bead we just added (for future linking) return(tpos); }; // Add sequence 'bid' to the multialign using ahang,bhang for positioning, and trace for the alignment. // // ahang can be negative, zero or positive. // // If negative, it will be equal in magnitude to bhang. // // If positive, bhang will be zero, and this is the amount of sequence in frank we should ignore. // So, we set apos to be this positive value. void abAbacus::applyAlignment(uint32 bid, int32 ahang, int32 UNUSED(bhang), int32 *trace, uint32 traceLen) { #ifdef DEBUG_ABACUS_ALIGN fprintf(stderr, "abAbacus::applyAlignment()-- ahang=%d traceLen=%u trace=%d %d %d %d %d ... %d %d %d %d %d\n", ahang, traceLen, (traceLen > 0) ? trace[0] : 0, (traceLen > 1) ? trace[1] : 0, (traceLen > 2) ? trace[2] : 0, (traceLen > 3) ? trace[3] : 0, (traceLen > 4) ? trace[4] : 0, (traceLen > 4) ? trace[traceLen-5] : 0, (traceLen > 3) ? trace[traceLen-4] : 0, (traceLen > 2) ? trace[traceLen-3] : 0, (traceLen > 1) ? trace[traceLen-2] : 0, (traceLen > 0) ? trace[traceLen-1] : 0); #endif // Finish some initialization. If this is the first call, readTofBead (and readTolBead) // will be NULL, and we need to allocate space for them. if (readTofBead == NULL) { readTofBead = new beadID [numberOfSequences()]; readTolBead = new beadID [numberOfSequences()]; } // Figure out where we are in the multialignment. abSequence *bseq = getSequence(bid); int32 alen = _columnsLen; // Actual length of the consensus sequence BEFORE any new bases are added. int32 blen = bseq->length(); // Actual length of the read we're adding. // We used to check that alen and blen were both positive, but we now allow applyAlignment() of // the first read (to an empty multialignment) to initialize the structure. int32 apos = MAX(ahang, 0); // if apos == alen, we'd just be pasting on new sequence. int32 bpos = 0; // if bpos == blen...we're pasting on one base? assert(apos <= alen); assert(0 <= bpos); // We tried letting bpos be set to non-zero, to ignore bases at the start of the read, assert(bpos <= blen); // but it doesn't work. abColumn *ncolumn = _columns[apos]; // The next empty column (where we will add the next aligned base). abColumn *pcolumn = NULL; // The previous column (that we just added a base to). uint16 plink = UINT16_MAX; // The read index of the read we just added to the previous column. beadID fBead; beadID lBead; // Negative ahang? Push these things onto the start of the multialign. // // We should fail if we get a negative ahang, but there is a column already before the first one // (which implies we aligned to something not full-length, possibly because we trimmed // frankenstein wrong).....but we don't even check. for (; bpos < -ahang; bpos++) { abColumn *newcol = new abColumn; plink = newcol->insertAtBegin(ncolumn, plink, bseq->getBase(bpos), bseq->getQual(bpos)); fBead.setF(newcol, plink); lBead.setL(newcol, plink); } if (ncolumn) pcolumn = ncolumn->_prevColumn; // Skip any positive traces at the start. These are template sequence (A-read) aligned to gaps // in the B-read, but we don't care because there isn't any B-read aligned yet. // // We probably should check that we didn't just process negative ahang above. for (; ((traceLen > 0) && (*trace != 0) && (*trace == 1)); trace++) { ncolumn = ncolumn->_nextColumn; apos++; } // Similarly, remove negative traces at the end. These are read sequence (B-read) aligned to gaps // in the A-read (template). The A-read extent should be reduced by one for each gap. while ((traceLen > 0) && (trace[traceLen-1] > blen)) { trace[--traceLen] = 0; } // Process the trace. while ((traceLen > 0) && (*trace != 0)) { // Gap is in afrag. Align ( - *trace - apos ) positions, then insert a new column, before // ncolumn, to accommodate the insert in B. ncolumn remains unchanged; it's still the column // that we want to place the next aligned base. if ( *trace < 0 ) { while (apos < (- *trace - 1)) { #ifdef DEBUG_ABACUS_ALIGN fprintf(stderr, "applyAlignment()-- align base %6d/%6d '%c' to column %7d\n", bpos, blen, bseq->getBase(bpos), ncolumn->position()); #endif plink = ncolumn->alignBead(plink, bseq->getBase(bpos), bseq->getQual(bpos)); fBead.setF(ncolumn, plink); lBead.setL(ncolumn, plink); pcolumn = ncolumn; // ...updating the previous column ncolumn = ncolumn->next(); // bpos++; // Move to the next base in the read. apos++; // Move to the next column } assert(apos < alen); assert(bpos < blen); // Three types of insert: // before the multialign (done above in the initialization) - an entirely new column, no gaps to add // after the multialign (done at the end) - same, entirely new column // insertion to the multialign - in the middle, complicated // // should it be insertAfter() to add a column before the exiting one and // appendColumn() to add a column after? appendColumm() is a special case of // tacking on new sequence to the multialign. // Add a new column for this insertion. abColumn *newcol = new abColumn; #ifdef DEBUG_ABACUS_ALIGN fprintf(stderr, "applyAlignment()-- align base %6d/%6d '%c' to after column %7d (new column)\n", bpos, blen, bseq->getBase(bpos), ncolumn->position()); #endif plink = newcol->insertAfter(pcolumn, plink, bseq->getBase(bpos), bseq->getQual(bpos)); fBead.setF(newcol, plink); lBead.setL(newcol, plink); pcolumn = newcol; bpos++; // Move to the next base in the read. } // Gap is in bfrag. Align ( *trace - bpos ) positions, then insert a gap bead into the read, // and align it to the existing column. if (*trace > 0) { while ( bpos < (*trace - 1) ) { #ifdef DEBUG_ABACUS_ALIGN fprintf(stderr, "applyAlignment()-- align base %6d/%6d '%c' to column %7d\n", bpos, blen, bseq->getBase(bpos), ncolumn->position()); #endif plink = ncolumn->alignBead(plink, bseq->getBase(bpos), bseq->getQual(bpos)); fBead.setF(ncolumn, plink); lBead.setL(ncolumn, plink); pcolumn = ncolumn; // ...updating the previous column ncolumn = ncolumn->next(); // bpos++; // Move to the next base in the read. apos++; // Move to the next column } assert(apos < alen); assert(bpos < blen); #ifdef DEBUG_ABACUS_ALIGN fprintf(stderr, "applyAlignment()-- align base %6d/%6d '-' to column %7d (gap in read)\n", bpos, blen, ncolumn->position()); #endif plink = ncolumn->alignBead(plink, '-', 0); fBead.setF(ncolumn, plink); lBead.setL(ncolumn, plink); pcolumn = ncolumn; ncolumn = ncolumn->next(); apos++; } trace++; } // Remaining alignment contains no indels, just slap in the bases. Note that when there is // no consensus sequence (this is the first read added) this loop does nothing; alen=apos=0. for (int32 rem = MIN(blen - bpos, alen - apos); rem > 0; rem--) { #ifdef DEBUG_ABACUS_ALIGN fprintf(stderr, "applyAlignment()-- align base %6d/%6d '%c' to column %7d (end of read)\n", bpos, blen, bseq->getBase(bpos), ncolumn->position()); #endif plink = ncolumn->alignBead(plink, bseq->getBase(bpos), bseq->getQual(bpos)); fBead.setF(ncolumn, plink); lBead.setL(ncolumn, plink); pcolumn = ncolumn; ncolumn = ncolumn->next(); apos++; bpos++; } // Finally, append any new (unaligned) sequence from the read. This handles the special case when there // are no existing columns (pcolumn == NULL). for (int32 rem=blen-bpos; rem > 0; rem--) { assert(ncolumn == NULL); // Can't be a column after where we're tring to append to! abColumn *newcol = new abColumn; #ifdef DEBUG_ABACUS_ALIGN fprintf(stderr, "applyAlignment()-- align base %6d/%6d '%c' to extend consensus\n", bpos, blen, bseq->getBase(bpos)); #endif plink = newcol->insertAtEnd(pcolumn, plink, bseq->getBase(bpos), bseq->getQual(bpos)); fBead.setF(newcol, plink); lBead.setL(newcol, plink); pcolumn = newcol; bpos++; #ifdef CHECK_LINKS newcol->checkLinks(); #endif } // Insert the first and last beads into our tracking maps. assert(fBead.column->_beads[fBead.link].prevOffset() == UINT16_MAX); assert(lBead.column->_beads[lBead.link].nextOffset() == UINT16_MAX); fbeadToRead[fBead] = bid; readTofBead[bid] = fBead; lbeadToRead[lBead] = bid; readTolBead[bid] = lBead; // Update the firstColumn in the abAbacus if it isn't set. updateColumns() will // reset it if the actual first column has changed here. if (_firstColumn == NULL) _firstColumn = fBead.column; // Finally, recall bases (not needed; done inline when bases are aligned) and refresh the column/cnsBases/cnsQuals lists. //recallBases(false); refreshColumns(); } canu-1.3/src/utgcns/libcns/abAbacus-baseCall.C000066400000000000000000000324211272335422300211650ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/BaseCall.C * src/AS_CNS/BaseCall.c * src/AS_CNS/MultiAlignment_CNS.c * src/utgcns/libcns/BaseCall.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-MAR-30 to 2008-FEB-13 * are Copyright 2005-2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Gennady Denisov from 2005-MAY-09 to 2008-JUN-06 * are Copyright 2005-2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2005-SEP-29 to 2006-OCT-03 * are Copyright 2005-2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-FEB-27 to 2009-MAY-14 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-JUL-28 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-NOV-23 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "abAbacus.H" #include using namespace std; void abColumn::baseCallMajority(void) { uint32 bsSum[CNS_NUM_SYMBOLS] = {0}; // Number of times we've seen this base uint32 qvSum[CNS_NUM_SYMBOLS] = {0}; // Sum of their QVs for (uint32 ii=0; ii<_beadsLen; ii++) { char base = _beads[ii].base(); uint8 qual = _beads[ii].qual(); uint32 bidx = baseToIndex[base]; if (bidx >= CNS_NUM_SYMBOLS) fprintf(stderr, "abColumn::baseCallMajority()-- For column %u, link %u, base '%c' (%d) is invalid.\n", position(), ii, _beads[ii].base(), _beads[ii].base()); assert(bidx < CNS_NUM_SYMBOLS); bsSum[bidx] += 1; qvSum[bidx] += qual; } // Find the best, and second best, ignore ties. uint32 bestIdx = 0; uint32 nextIdx = 1; for (uint32 i=1; i bsSum[bestIdx])) || ((bsSum[i] >= bsSum[bestIdx]) && (qvSum[i] > qvSum[bestIdx]))) { nextIdx = bestIdx; bestIdx = i; } else if (((bsSum[i] > bsSum[nextIdx])) || ((bsSum[i] >= bsSum[nextIdx]) && (qvSum[i] > qvSum[nextIdx]))) { nextIdx = i; } } // Original version set QV to zero. _call = indexToBase[bestIdx]; _qual = 0; // If the best is a gap, use the lowercase second best - the alignment will trest this specially if (_call == '-') _call = indexToBase[nextIdx] - 'A' + 'a'; } uint32 baseToTauIndex(char base) { switch (base) { case '-': return(0); break; case 'A': return(1); break; case 'C': return(2); break; case 'G': return(3); break; case 'T': return(4); break; default: fprintf(stderr, "baseToTauIndex()-- invalid base '%c' %d\n", base, base); assert(0); break; } return(0); } void abColumn::baseCallQuality(void) { char consensusBase = '-'; char consensusQual = 0; // The original versions classified reads as 'best allele', 'other allele' or 'guide allele'. // Other allele was set if we were targetting a specific set of reads here (e.g., we had already // clustered reads into haplotypes). Guide allele was used if the read was not really a read // (originally, these were reads that came from outside the project, but eventually, I think it // came to also mean the read was a unitig surrogate). All this was stripped out in early // December 2015. The pieces removed all mirrored what is done for the bReads. vector bReads; uint32 bBaseCount[CNS_NUM_SYMBOLS] = {0}; uint32 bQVSum[CNS_NUM_SYMBOLS] = {0}; // Best allele uint32 frag_cov = 0; // Scan a column of aligned bases. Sort into three groups: // - those corresponding to the reads of the best allele, // - those corresponding to the reads of the other allele and // - those corresponding to non-read fragments (aka guides) for (uint32 ii=0; ii<_beadsLen; ii++) { char base = _beads[ii].base(); uint8 qual = _beads[ii].qual(); uint32 bidx = baseToIndex[base]; // If we encode bases properly, this will go away. if (bidx >= CNS_NUM_SYMBOLS) fprintf(stderr, "abColumn::baseCallQuality()-- For column %u, link %u, base '%c' (%d) is invalid.\n", position(), ii, _beads[ii].base(), _beads[ii].base()); assert(bidx < CNS_NUM_SYMBOLS); bBaseCount[bidx] += 1; // Could have saved to 'best', 'other' or 'guide' here. bQVSum[bidx] += qual; bReads.push_back(ii); } double cw[5] = { 0.0, 0.0, 0.0, 0.0, 0.0 }; // "consensus weight" for a given base double tau[5] = { 1.0, 1.0, 1.0, 1.0, 1.0 }; // Compute tau based on real reads. for (uint32 cind=0; cind < bReads.size(); cind++) { char base = _beads[ bReads[cind] ].base(); uint8 qual = _beads[ bReads[cind] ].qual(); if (qual == 0) qual += 5; tau[0] += (base == '-') ? PROB[qual] : EPROB[qual]; tau[1] += (base == 'A') ? PROB[qual] : EPROB[qual]; tau[2] += (base == 'C') ? PROB[qual] : EPROB[qual]; tau[3] += (base == 'G') ? PROB[qual] : EPROB[qual]; tau[4] += (base == 'T') ? PROB[qual] : EPROB[qual]; //fprintf(stderr, "TAU7[%2d] %f %f %f %f %f qv %d\n", cind, tau[0], tau[1], tau[2], tau[3], tau[4], qual); consensusQual = qual; } //fprintf(stderr, "TAU8 %f %f %f %f %f\n", tau[0], tau[1], tau[2], tau[3], tau[4]); // Occasionally we get a single read of coverage, and the base is an N, which we ignored above. // This is probably of historical interest any more (it happened with 454 reads) but is left in // because its a cheap and working and safe. if (bReads.size() == 0) { // + oReads.size() + gReads.size() _call = 'N'; _qual = 0; return; } // Do we need to scale before normalizing? Anything out of bounds? We'll try to scale the small // values up to DBL_MIN without making the large values larger than DBL_MAX. If we end up with // some values still too small, oh well. We have a winner (the large value) anyway! // // Note, in log-space, min value (1e-309) is around -711.5, and max value (1e309) is around 711.5. //fprintf(stderr, "tau: %f %f %f %f %f\n", tau[0], tau[1], tau[2], tau[3], tau[4]); double minTau = DBL_MAX; double maxTau = -DBL_MAX; minTau = MIN(minTau, tau[0]); minTau = MIN(minTau, tau[1]); minTau = MIN(minTau, tau[2]); minTau = MIN(minTau, tau[3]); minTau = MIN(minTau, tau[4]); maxTau = MAX(maxTau, tau[0]); maxTau = MAX(maxTau, tau[1]); maxTau = MAX(maxTau, tau[2]); maxTau = MAX(maxTau, tau[3]); maxTau = MAX(maxTau, tau[4]); // Now that we know the min and max values, shift them as far positive as possible. Ideally, // this will just add an offset to bring the smallest value up to the minimum representable. double minValue = log(DBL_MIN) + DBL_EPSILON; double maxValue = log(DBL_MAX) - DBL_EPSILON; double scaleValue = 0.0; if (minTau < minValue) scaleValue = minValue - minTau; if (maxTau + scaleValue > maxValue) scaleValue = maxValue - maxTau; tau[0] += scaleValue; tau[1] += scaleValue; tau[2] += scaleValue; tau[3] += scaleValue; tau[4] += scaleValue; // It could however overflow the max (which is the value we care about), so any values still too // low are thresholded. if (tau[0] < minValue) tau[0] = minValue; if (tau[1] < minValue) tau[1] = minValue; if (tau[2] < minValue) tau[2] = minValue; if (tau[3] < minValue) tau[3] = minValue; if (tau[4] < minValue) tau[4] = minValue; //fprintf(stderr, "TAU9 %f %f %f %f %f value %f/%f tau %f/%f scale %f\n", // tau[0], tau[1], tau[2], tau[3], tau[4], minValue, maxValue, minTau, maxTau, scaleValue); // I give up. I can't make the following asserts true when the scale value is reset to not // exceed maxValue. I'm off, somewhere, by 1e-13 (EPSILON is 2e-16, nowhere near). My test case // takes 18 minutes to get here, and I don't really want to slap in a bunch of logging. So, // thresholding it is. if (tau[0] > maxValue) tau[0] = maxValue; if (tau[1] > maxValue) tau[1] = maxValue; if (tau[2] > maxValue) tau[2] = maxValue; if (tau[3] > maxValue) tau[3] = maxValue; if (tau[4] > maxValue) tau[4] = maxValue; assert(tau[0] <= maxValue); assert(tau[1] <= maxValue); assert(tau[2] <= maxValue); assert(tau[3] <= maxValue); assert(tau[4] <= maxValue); tau[0] = exp(tau[0]); tau[1] = exp(tau[1]); tau[2] = exp(tau[2]); tau[3] = exp(tau[3]); tau[4] = exp(tau[4]); //fprintf(stderr, "TAU10 %f %f %f %f %f\n", tau[0], tau[1], tau[2], tau[3], tau[4]); assert(tau[0] >= 0.0); assert(tau[1] >= 0.0); assert(tau[2] >= 0.0); assert(tau[3] >= 0.0); assert(tau[4] >= 0.0); cw[0] = tau[0] * 0.2; cw[1] = tau[1] * 0.2; cw[2] = tau[2] * 0.2; cw[3] = tau[3] * 0.2; cw[4] = tau[4] * 0.2; double normalize = cw[0] + cw[1] + cw[2] + cw[3] + cw[4]; double normalizeS = normalize; if (normalize > 0.0) { normalize = 1.0 / normalize; cw[0] *= normalize; cw[1] *= normalize; cw[2] *= normalize; cw[3] *= normalize; cw[4] *= normalize; } if ((cw[0] < 0.0) || (cw[1] < 0.0) || (cw[2] < 0.0) || (cw[3] < 0.0) || (cw[4] < 0.0)) fprintf(stderr, "ERROR: cw[0-4] invalid: %f %f %f %f %f - tau %f %f %f %f %f - normalize %.60e %.60e\n", cw[0], cw[1], cw[2], cw[3], cw[4], tau[0], tau[1], tau[2], tau[3], tau[4], normalize, normalizeS); if (!(cw[0] >= 0.0) || !(cw[1] >= 0.0) || !(cw[2] >= 0.0) || !(cw[3] >= 0.0) || !(cw[4] >= 0.0)) fprintf(stderr, "ERROR: cw[0-4] invalid: %f %f %f %f %f - tau %f %f %f %f %f - normalize %.60e %.60e\n", cw[0], cw[1], cw[2], cw[3], cw[4], tau[0], tau[1], tau[2], tau[3], tau[4], normalize, normalizeS); assert(cw[0] >= 0.0); assert(cw[1] >= 0.0); assert(cw[2] >= 0.0); assert(cw[3] >= 0.0); assert(cw[4] >= 0.0); double cwMax = DBL_MIN; uint32 cwIdx = 0; // Default is a gap. if (cw[0] > cwMax) { cwIdx = 0; cwMax = cw[0]; } // '-' if (cw[1] > cwMax) { cwIdx = 1; cwMax = cw[1]; } // 'A' if (cw[2] > cwMax) { cwIdx = 2; cwMax = cw[2]; } // 'C' if (cw[3] > cwMax) { cwIdx = 3; cwMax = cw[3]; } // 'G' if (cw[4] > cwMax) { cwIdx = 4; cwMax = cw[4]; } // 'T' // If cwMax = 0 then consensus is a gap. Otherwise, we deterministically set it to A C G T. #if 0 fprintf(stderr, "prob('%c') %f %c\n", indexToBase[0], cw[0], (cw[0] == cwMax) ? '*' : ' '); fprintf(stderr, "prob('%c') %f %c\n", indexToBase[1], cw[1], (cw[1] == cwMax) ? '*' : ' '); fprintf(stderr, "prob('%c') %f %c\n", indexToBase[2], cw[2], (cw[2] == cwMax) ? '*' : ' '); fprintf(stderr, "prob('%c') %f %c\n", indexToBase[3], cw[3], (cw[3] == cwMax) ? '*' : ' '); fprintf(stderr, "prob('%c') %f %c\n", indexToBase[4], cw[4], (cw[4] == cwMax) ? '*' : ' '); #endif consensusBase = indexToBase[cwIdx]; // Compute the QV. // If cwMax is big, we've max'd out the QV and set it to the maximum. // if (cwMax >= 1.0 - DBL_EPSILON) { consensusQual = CNS_MAX_QV; } // Otherwise compute the QV. If there is more than one read, or we used the surrogate, // we can compute from cwMax. If only one read, use its qv. // else { int32 qual = consensusQual; if ((bReads.size() > 1) /*|| (used_surrogate == true)*/) { double dqv = -10.0 * log10(1.0 - cwMax); qual = DBL_TO_INT(dqv); if (dqv - qual >= 0.50) qual++; } qual = MIN(CNS_MAX_QV, qual); qual = MAX(CNS_MIN_QV, qual); consensusQual = qual; } // If no target allele, or this is the target allele, set the base. Since there // is (currently) always no target allele, we always set the base. //fprintf(stderr, "SET %u to %c qv %c\n", position, consensusBase, consensusQV); _call = consensusBase; _qual = consensusQual; } char abColumn::baseCall(bool highQuality) { if (highQuality) baseCallQuality(); else baseCallMajority(); return(_call); } canu-1.3/src/utgcns/libcns/abAbacus-mergeRefine.C000066400000000000000000000246121272335422300217120ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MergeRefine.C * src/AS_CNS/MergeRefine.c * src/AS_CNS/MultiAlignment_CNS.c * src/utgcns/libcns/MergeRefine.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-MAR-30 to 2008-FEB-13 * are Copyright 2005-2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Gennady Denisov from 2005-MAY-09 to 2008-JUN-06 * are Copyright 2005-2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2005-SEP-29 to 2006-OCT-03 * are Copyright 2005-2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-FEB-27 to 2009-MAY-14 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-MAR-06 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-14 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "abAbacus.H" // Extends the read represented by column/beadLink into this column. uint16 abColumn::extendRead(abColumn *column, uint16 beadLink) { increaseArray(_beads, _beadsLen, _beadsMax, 1); uint32 link = _beadsLen++; _beads[link]._unused = column->_beads[beadLink]._unused; _beads[link]._isRead = column->_beads[beadLink]._isRead; _beads[link]._isUnitig = column->_beads[beadLink]._isUnitig; _beads[link]._base = '-'; _beads[link]._qual = 0; if (column->_beads[beadLink]._prevOffset == UINT16_MAX) { assert(column->_beads[beadLink]._nextOffset != UINT16_MAX); assert(_nextColumn == column); _beads[link]._prevOffset = UINT16_MAX; _beads[link]._nextOffset = beadLink; column->_beads[beadLink]._prevOffset = link; } else { assert(column->_beads[beadLink]._prevOffset != UINT16_MAX); assert(_prevColumn == column); _beads[link]._prevOffset = beadLink; _beads[link]._nextOffset = UINT16_MAX; column->_beads[beadLink]._nextOffset = link; } return(link); } // Merges the next column into this column, if possible. Possible if no read has // an actual base in both columns. bool abColumn::mergeWithNext(abAbacus *abacus, bool highQuality) { abColumn *lcolumn = this; abColumn *rcolumn = next(); abColumn *ncolumn = next()->next(); // The column after rcolumn. assert(lcolumn != NULL); assert(rcolumn != NULL); assert(lcolumn->next() == rcolumn); assert(rcolumn->prev() == lcolumn); #if 0 lcolumn->checkLinks(); rcolumn->checkLinks(); #endif // If both columns have a non-gap (for a single read), we cannot merge. for (uint32 ii=0; ii_beadsLen; ii++) { uint32 jj = lcolumn->_beads[ii].nextOffset(); if ((jj < UINT16_MAX) && (lcolumn->_beads[ii].base() != '-') && (rcolumn->_beads[jj].base() != '-')) return(false); } #if 0 fprintf(stderr, "MERGE columns %d %p <- %d %p\n", lcolumn->position(), lcolumn, rcolumn->position(), rcolumn); fprintf(stderr, "rcolumn links\n"); rcolumn->showLinks(); lcolumn->checkLinks(); rcolumn->checkLinks(); ncolumn->checkLinks(); #endif // OK to merge. Merge all the bases from the right column to the current column. We already // checked that whenever the right column has a base, the left column has a gap, so just march // down the right column and move those bases over! for (uint16 rr=0; rr_beadsLen; rr++) { uint16 ll = rcolumn->_beads[rr].prevOffset(); // Ignore the gaps. if (rcolumn->_beads[rr].base() == '-') continue; // Oh, great. We just found the end of a read. We need to link in the gap (in lcolumn) // before we can swap. Correction: we need to ADD a gap (in lcolumn) before we can swap. if (ll == UINT16_MAX) { //fprintf(stderr, "EXTEND READ at rr=%d\n", rr); ll = lcolumn->extendRead(rcolumn, rr); } // The simple case: just swap the contents. #if 0 fprintf(stderr, "mergeWithNext()-- swap beads lcolumn %d %c and rcolumn %d %c\n", ll, lcolumn->_beads[ll].base(), rr, rcolumn->_beads[rr].base()); #endif #ifdef BASECOUNT lcolumn->baseCountDecr(lcolumn->_beads[ll].base()); rcolumn->baseCountDecr(rcolumn->_beads[rr].base()); // We don't really care about rcolumn. #endif swap(lcolumn->_beads[ll], rcolumn->_beads[rr]); #ifdef BASECOUNT lcolumn->baseCountIncr(lcolumn->_beads[ll].base()); rcolumn->baseCountIncr(rcolumn->_beads[rr].base()); #endif // While we're here, update the bead-to-read maps. beadID oldb(rcolumn, rr); beadID newb(lcolumn, ll); map::iterator fit = abacus->fbeadToRead.find(oldb); // Does old bead exist map::iterator lit = abacus->lbeadToRead.find(oldb); // in either map? if (fit != abacus->fbeadToRead.end()) { uint32 rid = fit->second; //fprintf(stderr, "mergeWithNext()-- move fbeadToRead from %p/%d to %p/%d for read %d\n", // rcolumn, rr, lcolumn, ll, rid); abacus->fbeadToRead.erase(fit); // Remove the old bead to read pointer abacus->fbeadToRead[newb] = rid; // Add a new bead to read pointer abacus->readTofBead[rid] = newb; // Update the read to bead pointer } if (lit != abacus->lbeadToRead.end()) { uint32 rid = lit->second; //fprintf(stderr, "mergeWithNext()-- move lbeadToRead from %p/%d to %p/%d for read %d\n", // rcolumn, rr, lcolumn, ll, rid); abacus->lbeadToRead.erase(lit); abacus->lbeadToRead[newb] = rid; abacus->readTolBead[rid] = newb; } } // The rcolumn should now be full of gaps. (We could just test that baseCount('-') == depth() for (uint32 rr=0; rr_beadsLen; rr++) assert(rcolumn->_beads[rr].base() == '-'); #if 0 lcolumn->checkLinks(); rcolumn->checkLinks(); ncolumn->checkLinks(); #endif // To make checkLinks() work, we need to unlink rcolumn from the column list right now. if (rcolumn->_prevColumn) rcolumn->_prevColumn->_nextColumn = rcolumn->_nextColumn; if (rcolumn->_nextColumn) rcolumn->_nextColumn->_prevColumn = rcolumn->_prevColumn; assert(ncolumn == rcolumn->next()); assert(ncolumn == next()); // Before the rcolumn can be removed, we need to unlink it from the bead link list. If there is a column after rcolumn, // we need to move rcolumn's link pointers to lcolumn (prev) and ncolumn (next). // // The actual example (,'s indicate no bases because the read ended): // // 1234 Column 2 is merged into column 1, and then we delete column 2. // 1 -aa-aaa // 2 gaaT,,, Column 1 read 4 has _beads position 3. (next = 1) // 3 gaaT,,, Column 2 read 4 has _beads position 1. (prev = 3, next = 1) // 4 -aa-aaa Column 3 read 4 has _beads position 1. (prev = 1) // 5 -aa-aaa When column 2 is deleted, we're left with a busted link back from col 3 read 4; it should be 3 // 6 gaa,,,, if (ncolumn != NULL) { for (uint32 rr=0; rr_beadsLen; rr++) { uint16 bl = rcolumn->_beads[rr].prevOffset(); // back link from deleted column to lcolumn -> set as ncolumns back link (known as ll above) uint16 fl = rcolumn->_beads[rr].nextOffset(); // forw link from deleted column to ncolumn -> set as lcolumns forw link if (bl != UINT16_MAX) lcolumn->_beads[bl]._nextOffset = fl; if (fl != UINT16_MAX) ncolumn->_beads[fl]._prevOffset = bl; } } lcolumn->checkLinks(); ncolumn->checkLinks(); // Now, finally, we're done. Remove the old column, recall the base, and do a final check. //fprintf(stderr, "mergeWithNext()-- Remove rcolumn %d %p\n", rcolumn->position(), rcolumn); delete rcolumn; baseCall(highQuality); return(true); } // Simple sweep through the MultiAlignment columns, looking for columns to merge and removing null // columns // // Loop over all columns. If we do not merge, advance to the next column, otherwise, stay here and // merge to the now different next column (mergeCompatible removes the column that gets merged into // the current column). // // Note that _firstColumn is never removed. The second column could be merged into the first, // and the second one then removed. void abAbacus::mergeColumns(bool highQuality) { assert(_firstColumn != NULL); abColumn *column = _firstColumn; bool somethingMerged = false; assert(column->prev() == NULL); #if 0 fprintf(stderr, "mergeColumns()--\n"); display(stderr); #endif // If we merge, update the base call, and stay here to try another merge of the now different // next column. Otherwise, we didn't merge anything, so advance to the next column. while (column->next()) { if (column->mergeWithNext(this, highQuality) == true) somethingMerged = true; else column = column->next(); } // If any merges were performed, refresh. This updates the column list. if (somethingMerged) refreshColumns(); } canu-1.3/src/utgcns/libcns/abAbacus-refine.C000066400000000000000000001151501272335422300207300ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/AbacusRefine.C * src/AS_CNS/AbacusRefine.c * src/AS_CNS/MultiAlignment_CNS.c * src/utgcns/libcns/AbacusRefine.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-MAR-30 to 2008-FEB-13 * are Copyright 2005-2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Gennady Denisov from 2005-MAY-09 to 2008-JUN-06 * are Copyright 2005-2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2005-SEP-29 to 2006-OCT-03 * are Copyright 2005-2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-FEB-27 to 2010-JUN-09 * are Copyright 2008-2010 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2011-SEP-21 * are Copyright 2011 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-30 to 2015-AUG-14 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-10 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "abAbacus.H" #define MAX_WINDOW_FOR_ABACUS_REFINE 100 enum ShiftStatus { LEFT_SHIFT = (int) 'L', // Left Shifted (76) RIGHT_SHIFT = (int) 'R', // Right Shifted (82) UNSHIFTED = (int) 'U', // Unshifted (85) }; class abAbacusWork { public: abAbacusWork() { start_column = NULL; end_column = NULL; rows = 0; columns = 0; window_width = 0; shift = UNSHIFTED; beads = NULL; calls = NULL; }; abAbacusWork(abAbacus *abacus, abColumn *from, abColumn *end); ~abAbacusWork() { delete [] beads; delete [] calls; }; void reset(void) { for (uint32 ii=0; iistart_column = start_column; clone->end_column = end_column; clone->window_width = window_width; clone->rows = rows; clone->columns = columns; clone->shift = shift; clone->beads = new char [rows * (columns+2)]; clone->calls = new char [columns]; memcpy(clone->beads, beads, (rows * (columns+2)) * sizeof(char)); memcpy(clone->calls, calls, (columns) * sizeof(char)); clone->abacus_indices = abacus_indices; return(clone); }; void show(void) { char form[32]; sprintf(form, "'%%%d.%ds'\n", columns, columns); fprintf(stderr, "start_column 0x%16p %d\n", start_column, start_column->position()); fprintf(stderr, "end_column 0x%16p %d\n", end_column, end_column->position()); fprintf(stderr, "rows %d columns %d\n", rows, columns); fprintf(stderr, "window_width %d\n", window_width); fprintf(stderr, "shift %c\n", shift); // Shows the 'n' border around each row. for (int32 i=0; inumberOfSequences()]; start_column = bgn; end_column = end; rows = 0; // rows is incremented and set below window_width = end->position() - bgn->position() + 1; columns = 3 * window_width; shift = UNSHIFTED; for (uint32 ii=0; iinumberOfSequences(); ii++) { abacus_indices[ii] = UINT32_MAX; // Needs to use abAbacus readTofBead and readTolBead; see unitigConsensus::refreshPositions #if 0 if (abacus->getSequence(ii)->lastColumn()->position() < bgn->position()) continue; if (abacus->getSequence(ii)->firstColumn()->position() > end->position()) continue; #endif abacus_indices[ii] = rows++; } // Now we can allocate beads and calls. beads = new char [rows * (columns + 2)]; // two extra gap columns, plus "null" borders calls = new char [columns]; // Clear every base. for (uint32 ii=0; iinumberOfSequences(); ss++) { if (abacus_indices[ss] == UINT32_MAX) continue; // Read is in the range. We need to figure out a beadLink for some column, preferably close, // so that we can start iterating over the beads. abColumn *bColumn = bgn; uint16 bLink = UINT16_MAX; // Find the earliest column with a link map. The data for this is NOT IMPLEMENTED. #if 0 while (bColumn->beadReadIDsExist() == false) bColumn = bColumn->prev(); // Find the link, move back to the starting column for (uint32 ll=0; lldepth(); ll++) if (ss == bColumn->beadReadID(ll)) { bLink = ll; break; } #endif assert(bLink != UINT16_MAX); while (bColumn != bgn) { bLink = bColumn->bead(bLink)->nextOffset(); bColumn = bColumn->next(); } // Now just add bases into abacus. uint32 columnIndex = 0; while (1) { setBase(abacus_indices[ss], window_width + columnIndex, bColumn->bead(bLink)->base()); if (bColumn == end) break; bLink = bColumn->bead(bLink)->nextOffset(); bColumn = bColumn->next(); } } // Clear the border columns. This is done last, so that we can initialize beads with no read // coverage to 'n' (from the first initialization), and then clear the borders to '-' (here). for (uint32 i=0; i 0) && (cc < columns - 1) && ((getBase(rr, cc-1) == 'n') || (getBase(rr, cc+1) == 'n'))) b = 'n'; counts[b]++; } // Pick the majority. char best = 0; for (uint32 ii=0; ii<256; ii++) if (counts[best] < counts[ii]) best = ii; // Pick a call, and score the non-gap columns. if (counts['-'] + counts['n'] == rows) { calls[cc] = 'n'; } else { cols++; calls[cc] = best; score += rows - counts[best] - counts['n']; } } return(score); } uint32 abAbacusWork::affineScoreAbacus(void) { // This simply counts the number of opened gaps, to be used in tie breaker // of edit scores. uint32 score=0; uint32 start_column; uint32 end_column; if (shift == LEFT_SHIFT) { start_column = 0; end_column = columns/3; } else if (shift == RIGHT_SHIFT) { start_column = 2*columns/3; end_column = columns; } else { // shift == UNSHIFTED start_column = columns/3; end_column = 2*columns/3; } // Size of a gap does not matter, their number in a row does - GD for (uint32 i=0; i0;j--) { int32 null_column = 1; for (int32 i=0; i= 0 && num_gaps > num_ns) { columns_merged++; for (int32 i=0;i 0 */ { for (int32 j=last_non_null-1; j>first_non_null; j--) { int32 num_gaps=0, num_ns=0; mergeok = 1; curr_column_good = -1; for (int32 i=0;i= 0 && num_gaps > num_ns) { columns_merged++; for (int32 i=0;ifirst_non_null; k--) { prev = getBase(i,k-1); setBase(i, k, prev); } setBase(i, first_non_null, '-'); } // Return to the next column to see if it can be merged again j++; } } } return(columns_merged); } // lcols is the number of non-null columns in result uint32 abAbacusWork::leftShift(int32 &lcols) { //fprintf(stderr, "leftShift()--\n"); reset(); //show(); for (int32 j=window_width; j<2*window_width; j++) { for (int32 i=0; iwindow_width-1;j--) { for (int32 i=0; ij;pcol--) { char call = calls[pcol]; if (call != 'n' && call != c && c != 'n' ) continue; if (call == 'n') calls[pcol] = c; if (calls[pcol] == c || c == 'n' ) { setBase(i,j,'-'); setBase(i,pcol,c); break; } } if (getBase(i,j) != '-' ) calls[j] = c; } } } //fprintf(stderr, "rightShift()-- PREMERGE\n"); //show(); merge(1); //fprintf(stderr, "rightShift()-- POSTMERGE\n"); //show(); shift = RIGHT_SHIFT; return(scoreAbacus(rcols)); } #if 0 // Relationship must be one of: // // a) end gap moving left: // // X > A > B > C > ... > - becomes X - A B C ... // ^________________/ // // b) non-gap moving left across only gap characters // (more efficient special case, since first gap and last // character can just be exchanged) // // X > - > - > - > ... > A becomes X A - - - ... // ^________________/ static void leftEndShiftBead(abAbacus *abacus, abBeadID bid, abBeadID eid) { abBead *shift = abacus->getBead(eid); abBeadID aid = abacus->getBead(bid)->prevID(); assert(shift != NULL); if (abacus->getBase(shift->baseIdx()) != '-' ) { // assume first and internal characters are gaps abacus->lateralExchangeBead(bid, eid); } else { while (shift->prevID() != aid ) { abacus->lateralExchangeBead(shift->prevID(), shift->ident()); } } } // Relationship must be one of: // // a) end gap moving right: // // - > A > B > ... > C > X becomes A B ... C - X // \_________________^ // // b) non-gap moving right across only gap characters // (more efficient special case, since first gap and last // character can just be exchanged) // // A > - > - > ... > - > X becomes - - - ... A X // \________________^ static void rightEndShiftBead(abAbacus *abacus, abBeadID bid, abBeadID eid) { abBead *shift = abacus->getBead(bid); abBeadID aid = abacus->getBead(eid)->nextID(); assert(shift != NULL); if (abacus->getBase(shift->baseIdx()) != '-' ) { // assume last and internal characters are gaps abacus->lateralExchangeBead(bid, eid); } else { while (shift->nextID() != aid ) { abacus->lateralExchangeBead(shift->ident(), shift->nextID()); } } } void abAbacusWork::applyAbacus(abAbacus *abacus) { beadID bid; // ALWAYS the id of bead beadID eid; // ALWAYS the id of exch //fprintf(stderr, "applyAbacus()-- shift=%c start=%d width=%d\n", shift, start_column.get(), window_width); //show(); if (shift == LEFT_SHIFT) { abColumn *column = start_column; for (uint32 columnCount=0; columnCount < window_width; ) { bid.column = column; bid.link = 0; //fprintf(stderr, "0; bid=%d eid=%d\n", bid.get(), eid.get()); // Update all beads in a given column while (bid.link != UINT16_MAXisValid()) { bead = abacus->getBead(bid); char a_entry = getBase(abacus_indices[bead->seqIdx().get()] - 1, columnCount); //fprintf(stderr, "a_entry=%c bead=%c\n", a_entry, abacus->getBase(bead->baseIdx())); if (a_entry == 'n') { eid = bead->upID(); exch = abacus->getBead(eid); //fprintf(stderr, "1; bid=%d eid=%d\n", bid.get(), eid.get()); abacus->unalignTrailingGapBeads(bid); } else if (a_entry != abacus->getBase(bead->baseIdx())) { // Look for matching bead in frag and exchange eid = bead->ident(); exch = abacus->getBead(eid); //fprintf(stderr, "2; bid=%d eid=%d\n", bid.get(), eid.get()); if (NULL == exch) { eid = abacus->appendGapBead(bead->ident()); bead = abacus->getBead(bid); abacus->alignBeadToColumn(abacus->getColumn(bead->colIdx())->nextID(),eid, "ApplyAbacus(1)"); exch = abacus->getBead(eid); } //fprintf(stderr, "3; bid=%d eid=%d\n", bid.get(), eid.get()); while (a_entry != abacus->getBase(exch->baseIdx())) { abBeadID eidp = exch->nextID(); if (exch->nextID().isInvalid()) { eidp = abacus->appendGapBead(exch->ident()); bead = abacus->getBead(bid); exch = abacus->getBead(eid); abacus->alignBeadToColumn(abacus->getColumn(exch->colIdx())->nextID(),eidp, "ApplyAbacus(2)"); //fprintf(stderr, "4; bid=%d eid=%d\n", bid.get(), eid.get()); } else if (exch->colIdx() == end_column) { eidp = abacus->appendGapBead(exch->ident()); bead = abacus->getBead(bid); exch = abacus->getBead(eid); //fprintf(stderr, "5; bid=%d eid=%d\n", bid.get(), eid.get()); abColumn *cid = column; abacus->appendColumn(exch->colIdx(),eidp); column = abacus->getColumn(cid); } //fprintf(stderr, "6; bid=%d eid=%d b col/frg=%d/%d e_col/frg=%d/%d\n", // bid.get(), eid.get(), // bead->colIdx().get(), bead->seqIdx().get(), // exch->colIdx().get(), exch->seqIdx().get()); eid = eidp; exch = abacus->getBead(eid); } //fprintf(stderr,"LeftShifting bead %d (%c) with bead %d (%c).\n", // bid.get(), abacus->getBase(bead->baseIdx()), // eid.get(), abacus->getBase(exch->baseIdx())); leftEndShiftBead(abacus, bid, eid); } else { // no exchange necessary; eid = bid; exch = bead; //fprintf(stderr, "7; bid=%d eid=%d\n", bid.get(), eid.get()); } bid = exch->downID(); bead = NULL; //fprintf(stderr,"New bid is %d (%c), from %d down\n", // bid.get(), (bid.isValid()) ? abacus->getBase(abacus->getBead(bid)->baseIdx()) : 'n', // eid.get()); } // End of update; call base now. base = abacus->baseCall(column->ident(), true); column = column->next(); columnCount++; } } if (shift == RIGHT_SHIFT) { abColumn *column = end_column; for (uint32 columnCount=0; columnCount < window_width; ) { bid = abacus->getBead(column->callID())->downID(); while (bid.isValid()) { bead = abacus->getBead(bid); a_entry = getBase(abacus_indices[bead->seqIdx().get()] - 1, columns - columnCount - 1); if (a_entry == 'n') { eid = bead->upID(); exch = abacus->getBead(eid); abacus->unalignTrailingGapBeads(bid); } else if (a_entry != abacus->getBase(bead->baseIdx())) { // Look for matching bead in frag and exchange eid = bead->ident(); exch = abacus->getBead(eid); if (NULL == exch) { eid = abacus->prependGapBead(bead->ident()); bead = abacus->getBead(bid); exch = abacus->getBead(eid); abacus->alignBeadToColumn(abacus->getColumn(bead->colIdx())->prevID(),eid, "ApplyAbacus(3)"); } while (a_entry != abacus->getBase(exch->baseIdx())) { abBeadID eidp = exch->prevID(); if (exch->prevID().isInvalid()) { eidp = abacus->prependGapBead(exch->ident()); bead = abacus->getBead(bid); exch = abacus->getBead(eid); abacus->alignBeadToColumn(abacus->getColumn(exch->colIdx())->prevID(),eidp, "ApplyAbacus(4)"); } else if (exch->colIdx() == start_column) { eidp = abacus->appendGapBead(exch->prevID()); bead = abacus->getBead(bid); exch = abacus->getBead(eid); abColumn *cid = column; abacus->appendColumn(abacus->getColumn(exch->colIdx())->prevID(), eidp); column = abacus->getColumn(cid); } eid = eidp; exch = abacus->getBead(eid); } //fprintf(stderr,"RightShifting bead %d (%c) with bead %d (%c).\n", // eid.get(), abacus->getBase(exch->baseIdx()), // bid.get(), abacus->getBase(bead->baseIdx())); rightEndShiftBead(abacus, eid, bid); } else { eid = bid; exch = bead; // no exchange necessary; } bid = exch->downID(); bead = NULL; //fprintf(stderr,"New bid is %d (%c), from %d down\n", // bid.get(), (bid.isValid()) ? abacus->getBase(abacus->getBead(bid)->baseIdx()) : 'n', // eid.get()); } base = abacus->baseCall(column->ident(), true); column = abacus->getColumn(column->prevID()); columnCount++; } } } #endif void abAbacusWork::applyAbacus(abAbacus *abacus) { } // // In this case, we just look for a string of gaps in the consensus sequence // static int32 IdentifyWindow_Smooth(abColumn *&bgnCol, abColumn *&terCol) { terCol = bgnCol->next(); // Consensus not a gap, nothing to do. if (bgnCol->baseCall() != '-') return(0); int32 winLen = 1; // Consensus is a gap. Expand it to the maximum gap. while ((terCol->baseCall() == '-') && (terCol->next() != NULL)) { terCol = terCol->next(); winLen++; } #ifdef DEBUG_IDENTIFY_WINDOW fprintf(stderr, "identifyWindow()-- gap at %d to %d winLen=%d (return)\n", bgnCol->position(), ter->position(), winLen); #endif return(winLen); } // // Here, we're looking for a string of the same character // static int32 IdentifyWindow_Poly_X(abColumn *&bgnCol, abColumn *&terCol) { terCol = bgnCol->next(); int32 gapCount = bgnCol->baseCount('-'); int32 winLen = 1; char poly = bgnCol->baseCall(); if (poly == '-') return(0); while ((terCol->baseCall() == poly) || (terCol->baseCall() == '-')) { if (terCol->next() == NULL) break; gapCount += terCol->baseCount('-'); winLen += 1; terCol = terCol->next(); } if (winLen <= 2) return(0); // capture trailing gap-called columns while (terCol->baseCall() == '-' ) { if (terCol->majorityBase(true) != poly) break; if (terCol->next() == NULL) break; gapCount += terCol->baseCount('-'); winLen += 1; terCol = terCol->next(); } // now that a poly run with trailing gaps is established, look for leading gaps abColumn *preCol = bgnCol->prev(); while (preCol != NULL) { if ((preCol->baseCall() != '-') && (preCol->baseCall() != poly)) break; bgnCol = preCol; gapCount += preCol->baseCount('-'); winLen += 1; preCol = preCol->prev(); } //fprintf(stderr,"POLYX candidate (%c) at column %d ter %d , width %d, gapcount %d\n", // poly, bgnCol->position(), terCol.get(), winLen, gapCount); if ((bgnCol->prev() != NULL) && (winLen > 2) && (gapCount > 0)) return(winLen); return(0); } // // In this case, we look for a string mismatches, indicating a poor alignment region // which might benefit from Abacus refinement // // heuristics: // // > terle border on either side of window of width: STABWIDTH // > fewer than STABMISMATCH in stable border // // _ __ ___ // SSSSS SSSSS SSSSS .SSSS+ SSSSS .SSSS+ // SSSSS SSSSS SSSSS .SSSS+ SSSSS .SSSS+ // SSSSS SSSSS => SSSSS .SSSS+ => SSSSS .SSSS+ // SSSSS SSSSS SSSSS .SSSS+ SSSSS .SSSS+ // SSSSS_SSSSS SSSSS_.SSSS+ SSSSS__.SSSS+ // | | | // |\_____________|_______________|____ growing 'gappy' window // | // bgnCol // #define STABWIDTH 6 static int32 IdentifyWindow_Indel(abColumn *&bgnCol, abColumn *&terCol) { terCol = bgnCol->next(); int32 cum_mm = bgnCol->mismatchCount(); int32 stab_mm = 0; int32 stab_gaps = 0; int32 stab_width = 0; int32 stab_bases = 0; int32 winLen = 0; if ((cum_mm == 0) || (bgnCol->baseCount('-') == 0)) return(0); abColumn *stableEnd = terCol; // Compute the number of mismatches, gapes and bases in the next STABWIDTH columnd. while ((stableEnd->next() != NULL) && (stab_width < STABWIDTH)) { stab_mm += stableEnd->mismatchCount(); stab_gaps += stableEnd->baseCount('-'); stab_bases += stableEnd->depth(); stableEnd = stableEnd->next(); stab_width++; } // If no bases (how?) return an empty window. if (stab_bases == 0) return(0); // While the number of mismatches is high, and the number of gaps is also high, shift the stable // region ahead by one column. Subtract out the values for the column we shift out (on the left) // and add in the values for the column we shift in (on the right). while ((stab_mm > 0.02 * stab_bases) || // CNS_SEQUENCING_ERROR_EST (stab_gaps > 0.25 * stab_bases)) { int32 mm = terCol->mismatchCount(); int32 gp = terCol->baseCount('-'); int32 bps = terCol->depth(); // move terCol ahead if (stableEnd->next() != NULL) { stab_mm += stableEnd->mismatchCount(); stab_bases += stableEnd->depth(); stab_gaps += stableEnd->baseCount('-'); stableEnd = stableEnd->next(); stab_mm -= mm; stab_gaps -= gp; stab_bases -= bps; cum_mm += mm; terCol = terCol->next(); winLen++; } else { break; } } if (winLen > 1) return(winLen); return(0); } int32 abAbacus::refineWindow(abColumn *bgnCol, abColumn *terCol) { abAbacusWork *orig_abacus = new abAbacusWork(this, bgnCol, terCol); int32 orig_columns = 0; int32 orig_mm_score = orig_abacus->scoreAbacus(orig_columns); abAbacusWork *left_abacus = orig_abacus->clone(); int32 left_columns = 0; abAbacusWork *right_abacus = orig_abacus->clone(); int32 right_columns = 0; int32 left_mm_score = left_abacus->leftShift(left_columns); int32 right_mm_score = right_abacus->rightShift(right_columns); // determine best score and apply abacus to real columns int32 orig_gap_score = orig_abacus->affineScoreAbacus(); int32 left_gap_score = left_abacus->affineScoreAbacus(); int32 right_gap_score = right_abacus->affineScoreAbacus(); abAbacusWork *best_abacus = orig_abacus; int32 best_columns = orig_columns; int32 best_gap_score = orig_gap_score; int32 best_mm_score = orig_mm_score; int32 orig_total_score = orig_mm_score + orig_columns + orig_gap_score; int32 left_total_score = left_mm_score + left_columns + left_gap_score; int32 right_total_score = right_mm_score + right_columns + right_gap_score; int32 best_total_score = orig_total_score; int32 score_reduction = 0; // Use the total score to refine the abacus if (left_total_score < orig_total_score || right_total_score < orig_total_score ) { if (left_total_score <= right_total_score ) { score_reduction += orig_total_score - left_total_score; //fprintf(stderr,"\nTry to apply LEFT abacus:\n"); //ShowAbacus(left_abacus); best_abacus = left_abacus; best_mm_score = left_mm_score; best_columns = left_columns; best_gap_score = left_gap_score; best_total_score = left_total_score; } else { score_reduction += orig_total_score - right_total_score; //fprintf(stderr,"\nTry to apply RIGHT abacus:\n"); //ShowAbacus(right_abacus); best_abacus = right_abacus; best_mm_score = right_mm_score; best_columns = right_columns; best_gap_score = right_gap_score; best_total_score = right_total_score; } } best_abacus->applyAbacus(this); delete orig_abacus; delete left_abacus; delete right_abacus; return(score_reduction); } //********************************************************************************* // Abacus Refinement: // AbacusRefine contains the logic for sweeping through the multialignment, // and identifying candidate windows for refinement. // Each window is cast into an abacus, which is left and right shifted. // The best resulting base arrangement (if different from input) is then // applied to window of the MultiAlignment //********************************************************************************* // AbacusRefine // // ctgcns - from=0 to=-1 level=abAbacus_Smooth // - from=0 to=-1 level=abAbacus_Poly_X // - from=0 to=-1 level=abAbacus_Indel // utgcns - same // // from,to are C-style. Used to be INCLUSIVE, but never used anyway // int32 abAbacus::refine(abAbacusRefineLevel level, uint32 bgn, uint32 end) { #warning SKIPPING ALL REFINEMENTS return(0); if (end > numberOfColumns()) end = numberOfColumns(); abColumn *bgnCol = _columns[bgn]; abColumn *endCol = _columns[end - 1]; abColumn *terCol = NULL; int32 score_reduction = 0; while (bgnCol != endCol) { int32 window_width = 0; switch (level) { case abAbacus_Smooth: window_width = IdentifyWindow_Smooth(bgnCol, terCol); break; case abAbacus_Poly_X: window_width = IdentifyWindow_Poly_X(bgnCol, terCol); break; case abAbacus_Indel: window_width = IdentifyWindow_Indel(bgnCol, terCol); break; default: break; } // If the window is too big, there's likely a polymorphism that won't respond well to abacus, // so skip it. // if ((window_width > 0) && (window_width < MAX_WINDOW_FOR_ABACUS_REFINE)) { // Insert a gap column for maneuvering room if the window starts in the first. This used to // be logged, and BPW can't remember EVER seeing the message. #if 0 if (bgnCol->prev() == NULL) { abBeadID firstbeadID = abacus->getBead(bgnCol->callID() )->downID(); abBeadID newbeadID = abacus->appendGapBead(abacus->getBead(firstbeadID)->ident()); fprintf(stderr, "abMultiAlign::refine()-- Adding gap bead "F_U32" after first bead "F_U32" to add abacus room for abutting left of multialignment\n", newbeadID.get(), firstbeadID.get()); abacus->appendColumn(abacus->getBead(firstbeadID)->colIdx(), newbeadID); } #endif // Actually do the refinements. score_reduction += refineWindow(bgnCol, terCol); } // Move to the column after the window we just examined. bgnCol = terCol; } // WITH quality=1 make_v_list=1, all the rest defaults refreshColumns(); recallBases(true); return(score_reduction); } canu-1.3/src/utgcns/libcns/abAbacus-refreshMultiAlign.C000066400000000000000000000117031272335422300231030ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignment_CNS.c * src/AS_CNS/RefreshMANode.C * src/AS_CNS/RefreshMANode.c * src/utgcns/libcns/RefreshMANode.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-MAR-30 to 2008-FEB-13 * are Copyright 2005-2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Gennady Denisov from 2005-MAY-09 to 2008-JUN-06 * are Copyright 2005-2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2005-SEP-29 to 2006-OCT-03 * are Copyright 2005-2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-FEB-27 to 2009-SEP-25 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-MAR-06 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-14 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "abAbacus.H" // (Optionally) recall consensus in all columns. // Rebuild the list of columns in the multialign. // Rebuild the column to position map. void abAbacus::refreshColumns(void) { //fprintf(stderr, "abAbacus::refreshColumns()--\n"); // Given that _firstColumn is a valid column, walk to the start of the column list. while (_firstColumn->_prevColumn != NULL) _firstColumn = _firstColumn->_prevColumn; // Number the columns, so we can make sure the _columns array has enough space. Probably not // needed to be done first, but avoids having the resize call in the next loop. uint32 cn = 0; for (abColumn *column = _firstColumn; column; column = column->next()) column->_columnPosition = cn++; // Position of the column in the gapped consensus. // Fake out resizeArray so it will work on three arrays. uint32 cm = _columnsMax; resizeArray(_columns, 0, cm, cn+1, resizeArray_doNothing); cm = _columnsMax; resizeArray(_cnsBases, 0, cm, cn+1, resizeArray_doNothing); cm = _columnsMax; resizeArray(_cnsQuals, 0, cm, cn+1, resizeArray_doNothing); _columnsMax = cm; // Build the list of columns and update consensus and quals while we're there. _columnsLen = 0; for (abColumn *column = _firstColumn; column; column = column->next()) { _columns [_columnsLen] = column; _cnsBases[_columnsLen] = column->baseCall(); _cnsQuals[_columnsLen] = column->baseQual(); _columnsLen++; } _cnsBases[_columnsLen] = 0; _cnsQuals[_columnsLen] = 0; // Not actually zero terminated. //for (abColumn *column = _firstColumn; column; column = column->next()) // fprintf(stderr, "refreshColumns()-- column %p is at position %d\n", // column, column->position()); } void abAbacus::recallBases(bool highQuality) { //fprintf(stderr, "abAbacus::recallBases()-- highQuality=%d\n", highQuality); // Given that _firstColumn is a valid column, walk to the start of the column list. // We could use _columns[] instead. while (_firstColumn->_prevColumn != NULL) _firstColumn = _firstColumn->_prevColumn; // Number the columns, so we can make sure the _columns array has enough space. Probably not // needed to be done first, but avoids having the resize call in the next loop. for (abColumn *column = _firstColumn; column; column = column->next()) column->baseCall(highQuality); // After calling bases, we need to refresh to copy the bases from each column into // _cnsBases and _cnsQuals. refreshColumns(); } canu-1.3/src/utgcns/libcns/abAbacus.C000066400000000000000000000104641272335422300174640ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignment_CNS.C * src/AS_CNS/MultiAlignment_CNS.c * src/utgcns/libcns/MultiAlignment_CNS.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-MAR-30 to 2008-FEB-13 * are Copyright 2005-2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Gennady Denisov from 2005-MAY-09 to 2008-JUN-06 * are Copyright 2005-2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2005-SEP-29 to 2006-OCT-03 * are Copyright 2005-2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-FEB-27 to 2009-MAY-14 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-JUL-08 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-14 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "abAbacus.H" // Shouldn't be global, but some things -- like abBaseCount -- need it. bool DATAINITIALIZED = false; double EPROB[CNS_MAX_QV - CNS_MIN_QV + 1] = { 0 }; double PROB [CNS_MAX_QV - CNS_MIN_QV + 1] = { 0 }; uint32 baseToIndex[256] = { 0 }; char indexToBase[CNS_NUM_SYMBOLS] = { 0 }; void abAbacus::initializeGlobals(void) { for (int32 i=0; i<256; i++) baseToIndex[i] = UINT32_MAX; indexToBase[ 0] = '-'; indexToBase[ 1] = 'A'; indexToBase[ 2] = 'C'; indexToBase[ 3] = 'G'; indexToBase[ 4] = 'T'; indexToBase[ 5] = 'N'; #if 0 indexToBase[ 6] = 'a'; // -A indexToBase[ 7] = 'c'; // -C indexToBase[ 8] = 'g'; // -G indexToBase[ 9] = 't'; // -T indexToBase[10] = 'M'; // AC indexToBase[11] = 'R'; // AG indexToBase[12] = 'W'; // AT indexToBase[13] = 'S'; // CG indexToBase[14] = 'Y'; // CT indexToBase[15] = 'K'; // GT indexToBase[16] = 'm'; // -AC indexToBase[17] = 'r'; // -AG indexToBase[18] = 'w'; // -AT indexToBase[19] = 's'; // -CG indexToBase[20] = 'y'; // -CT indexToBase[21] = 'k'; // -GT indexToBase[22] = 'V'; // ACG indexToBase[23] = 'H'; // ACT indexToBase[24] = 'D'; // AGT indexToBase[25] = 'B'; // CGT indexToBase[26] = 'v'; // -ACG indexToBase[27] = 'h'; // -ACT indexToBase[28] = 'd'; // -AGT indexToBase[29] = 'b'; // -CGT indexToBase[30] = 'X'; // ACGT indexToBase[31] = 'x'; // -ACGT #endif for (int32 i=0; ibead(link)->base()); }; uint16 prevLink(void) { return(column->bead(link)->prevOffset()); }; uint16 nextLink(void) { return(column->bead(link)->nextOffset()); }; abColumn *prevColumn(void) { return(column->prev()); }; abColumn *nextColumn(void) { return(column->next()); }; abColumn *column; uint16 link; }; class abAbacus { public: abAbacus() { _sequencesLen = 0; _sequencesMax = 65536; _sequences = new abSequence * [_sequencesMax]; memset(_sequences, 0, sizeof(abSequence *) * _sequencesMax); _columnsLen = 0; _columnsMax = 1024 * 1024; _columns = new abColumn * [_columnsMax]; memset(_columns, 0, sizeof(abColumn *) * _columnsMax); _cnsBases = new char [_columnsMax]; _cnsQuals = new uint8 [_columnsMax]; memset(_cnsBases, 0, sizeof(char) * _columnsMax); memset(_cnsQuals, 0, sizeof(uint8) * _columnsMax); _firstColumn = NULL; readTofBead = NULL; readTolBead = NULL; if (DATAINITIALIZED == false) initializeGlobals(); }; ~abAbacus() { for (uint32 ss=0; ss<_sequencesLen; ss++) delete _sequences[ss]; for (abColumn *del = _firstColumn; (del = _firstColumn); ) { _firstColumn = _firstColumn->next(); delete del; } delete [] _sequences; delete [] _columns; delete [] _cnsBases; delete [] _cnsQuals; delete [] readTofBead; delete [] readTolBead; }; private: void initializeGlobals(void); public: char *bases(void) { return(_cnsBases); }; uint8 *quals(void) { return(_cnsQuals); }; // Adds gkpStore read 'readID' to the abacus; former AppendFragToLocalStore void addRead(gkStore *gkpStore, uint32 readID, uint32 askip, uint32 bskip, bool complemented, map *inPackageRead, map *inPackageReadData); public: void refreshColumns(void); void recallBases(bool highQuality = false); void appendBases(uint32 bid, uint32 bgn, uint32 end); void applyAlignment(uint32 bid, int32 ahang, int32 bhang, int32 *trace, uint32 traceLen); public: uint32 numberOfSequences(void) { return(_sequencesLen); }; abSequence *getSequence(uint32 sid) { assert(sid < _sequencesLen); return(_sequences[sid]); }; private: uint32 _sequencesLen; uint32 _sequencesMax; abSequence **_sequences; // The consensus sequence (actually, the multialign structure itself) Bases in columns get // shifted, but columns themselves do not move. However, we can insert columns. This array is // an in order list of the columns in the multialignment. public: abColumn *getFirstColumn(void) { return(_columns[0]); }; abColumn *getLastColumn(void) { return(_columns[_columnsLen-1]); }; uint32 numberOfColumns(void) { return(_columnsLen); }; abColumn *getColumn(uint32 cid) { assert(cid < _columnsLen); return(_columns[cid]); }; // Returns column position for position 'pos' in sequence 'seqIdx'. If the position is not contained // in the sequence, the remaining bases are assumed to be ungapped. uint32 getColumn(uint32 seqIdx, uint32 pos) { beadID bid = readTofBead[seqIdx]; uint32 cur = 0; while ((cur < pos) && (bid.nextLink() != UINT16_MAX)) { if (bid.base() != '-') // Count it only if it is a real base in the read. cur++; bid.link = bid.nextLink(); bid.column = bid.nextColumn(); } assert(bid.column != NULL); return(bid.column->position() + pos - cur); }; public: uint32 _columnsLen; uint32 _columnsMax; abColumn **_columns; // Sized in refreshColumns() char *_cnsBases; uint8 *_cnsQuals; abColumn *_firstColumn; public: // These maps are used to populate abSequence's first and last column pointers. // readToBead is the one we use for this. // beadToRead is used to ...? beadID *readTofBead; // Allocated once, after all reads are beadID *readTolBead; // added to us. map fbeadToRead; map lbeadToRead; // This is the former abMultiAlign private: //abBeadID unalignBeadFromColumn(abBeadID bid); //void unalignTrailingGapBeads(abBeadID bid); //void checkColumnBaseCounts(); //void lateralExchangeBead(abBeadID lid, abBeadID rid); public: void mergeColumns(bool highQuality); void getConsensus(tgTig *tig); uint32 getSequenceDeltas(uint32 sid, int32 *deltas); void getPositions(tgTig *tig); int32 refineWindow(abColumn *bgnCol_column, abColumn *terCol); int32 refine(abAbacusRefineLevel level, uint32 from = 0, uint32 to = UINT32_MAX); // There are two multiAlign displays; this one, and one in tgTig. void display(FILE *F); }; #endif canu-1.3/src/utgcns/libcns/abBead.H000066400000000000000000000103471272335422300171260ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignment_CNS.h * src/AS_CNS/MultiAlignment_CNS_private.H * src/AS_CNS/MultiAlignment_CNS_private.h * src/utgcns/libcns/MultiAlignment_CNS_private.H * * Modifications by: * * Gennady Denisov from 2005-MAY-23 to 2007-OCT-25 * are Copyright 2005-2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUL-08 to 2013-AUG-01 * are Copyright 2005-2009,2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2006-FEB-13 to 2008-FEB-13 * are Copyright 2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-JAN-28 to 2009-SEP-25 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2011-SEP-21 * are Copyright 2011 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-JAN-27 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-18 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef ABBEAD_H #define ABBEAD_H class abBead { public: abBead() { clear(); }; ~abBead() { }; void clear(void) { _unused = 0; _isRead = 0; _isUnitig = 0; _base = '-'; _qual = 0; _prevOffset = UINT16_MAX; _nextOffset = UINT16_MAX; }; void initialize(char base, uint8 qual, uint16 prevOff, uint16 nextOff) { _base = base; _qual = qual; _prevOffset = prevOff; _nextOffset = nextOff; }; public: // ACCESSORS char base(void) { return(_base); }; // Set only at init, or maybe char qual(void) { return(_qual); }; // reset with swap() uint16 prevOffset(void) { return(_prevOffset); }; // Generally set only at init, uint16 nextOffset(void) { return(_nextOffset); }; // columns are added public: // METHODS private: uint16 _unused:1; uint16 _isRead:1; // If set, base is from an actual read, use it for consensus. uint16 _isUnitig:1; // If set, base is from a unitig, don't use it for consensus (unless needed). uint16 _base:7; // Base at this position. (eventually will be encoded to 3 bits) uint16 _qual:6; // Quality at this position. uint16 _prevOffset; // Position in the array of beads for the previous column uint16 unused_thisOffset; // Position in the arary of beads for this column uint16 _nextOffset; // Position in the array of beads for the next column friend class abAbacus; friend class abColumn; // inferPrevNextBeadPointers(), mergeWithNext() friend void swap(abBead &a, abBead &b); }; inline void swap(abBead &a, abBead &b) { abBead c; c._isRead = b._isRead; c._isUnitig = b._isUnitig; c._base = b._base; c._qual = b._qual; b._isRead = a._isRead; b._isUnitig = a._isUnitig; b._base = a._base; b._qual = a._qual; a._isRead = c._isRead; a._isUnitig = c._isUnitig; a._base = c._base; a._qual = c._qual; } #endif // ABBEAD_H canu-1.3/src/utgcns/libcns/abColumn.C000066400000000000000000000047231272335422300175240ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignment_CNS.C * src/AS_CNS/MultiAlignment_CNS.c * src/utgcns/libcns/MultiAlignment_CNS.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-MAR-30 to 2008-FEB-13 * are Copyright 2005-2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Gennady Denisov from 2005-MAY-09 to 2008-JUN-06 * are Copyright 2005-2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2005-SEP-29 to 2006-OCT-03 * are Copyright 2005-2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-FEB-27 to 2009-MAY-14 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-JAN-21 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-18 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "abAbacus.H" canu-1.3/src/utgcns/libcns/abColumn.H000066400000000000000000000221321272335422300175230ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignment_CNS.h * src/AS_CNS/MultiAlignment_CNS_private.H * src/AS_CNS/MultiAlignment_CNS_private.h * src/utgcns/libcns/MultiAlignment_CNS_private.H * * Modifications by: * * Gennady Denisov from 2005-MAY-23 to 2007-OCT-25 * are Copyright 2005-2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUL-08 to 2013-AUG-01 * are Copyright 2005-2009,2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2006-FEB-13 to 2008-FEB-13 * are Copyright 2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-JAN-28 to 2009-SEP-25 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2011-SEP-21 * are Copyright 2011 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-JUL-01 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-18 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef ABCOLUMN_H #define ABCOLUMN_H #undef BASECOUNT // Used in abAbacus-refine.C, which is disabled. #include "abBead.H" class abAbacus; class abColumn { public: abColumn() { _columnPosition = INT32_MAX; _call = '-'; _qual = 0; _prevColumn = NULL; _nextColumn = NULL; _beadsMax = 0; _beadsLen = 0; _beads = NULL; #if 0 _beadReadIDs = NULL; #endif #ifdef BASECOUNT for (int32 ii=0; ii= _baseCounts[iMax]) iMax = i; return(indexToBase[iMax]); }; uint32 majorityCount(bool ignoreGap) { return(_baseCounts[majorityBase(ignoreGap)]); }; uint32 mismatchCount(void) { return(depth() - majorityCount(false)); } #else uint32 baseCount(char ) { return(0); }; // Just a stub so abAbacus-refine.C will compile char majorityBase(bool ) { return(0); }; uint32 mismatchCount(void) { return(0); }; #endif char baseCall(void) { return(_call); }; char baseQual(void) { return(_qual); }; abColumn *prev(void) { return(_prevColumn); }; abColumn *next(void) { return(_nextColumn); }; private: void allocateInitialBeads(void); void inferPrevNextBeadPointers(void); public: uint16 insertAtBegin(abColumn *first, uint16 prevLink, char base, uint8 qual); uint16 insertAtEnd (abColumn *prev, uint16 prevLink, char base, uint8 qual); uint16 insertAfter (abColumn *prev, uint16 prevLink, char base, uint8 qual); uint16 alignBead(uint16 prevIndex, char base, uint8 qual); uint16 extendRead(abColumn *column, uint16 beadLink); bool mergeWithNext(abAbacus *abacus, bool highQuality); private: void baseCallMajority(void); void baseCallQuality(void); public: char baseCall(bool highQuality); public: abColumn *mergeNext(void); private: int32 _columnPosition; // coordinate of the column in the multialign; needs to be signed for compat with tgTig bgn/end char _call; // The base call for this column. uint8 _qual; // The quality of that base call. // 16 bytes of pointers. // Alternate schemes: // two 4 byte offsets into an arary of pointers, but that's also 16 bytes per column. // two 4 byte offsets into an array of objects, but we'd then need to realloc sometime. // two 4 byte offsets into a heap, but that adds a lot of complication. // abColumn *_prevColumn; abColumn *_nextColumn; public: abBead *bead(uint32 ii) { return(_beads + ii); }; private: uint16 _beadsMax; // Number of beads allocated uint16 _beadsLen; // Depth; number of reads that span this column abBead *_beads; // If allocated, the read idx (NOT gkpID) for each bead in the column. This will // be used to (efficiently) map arbitrary columns back to their reads when refining abacus // layouts. The data is never populated (needs to be done in applyAlignment()). // #if 0 public: bool beadReadIDsExist(void) { return(_beadReadIDs != NULL); }; uint32 beadReadID(uint32 xx) { return(_beadReadIDs[xx]); }; private: uint32 *_beadReadIDs; #endif #ifdef BASECOUNT uint16 _baseCounts[CNS_NUM_SYMBOLS]; #endif public: void checkLinks(void) { uint32 nErrors = 0; for (uint32 tt=0; tt<_beadsLen; tt++) { uint16 pl = _beads[tt].prevOffset(); uint16 nl = _beads[tt].nextOffset(); if ((prev() != NULL) && (pl != UINT16_MAX) && (prev()->_beads[pl].nextOffset() != tt)) fprintf(stderr, "ERROR: prev bead pl=%d nextOffset=%d != tt=%d\n", pl, prev()->_beads[pl].nextOffset(), tt), nErrors++; if ((prev() == NULL) && (pl != UINT16_MAX)) fprintf(stderr, "no prev column, yet defined link for column %p bead %d\n", this, tt), nErrors++; if ((next() != NULL) && (nl != UINT16_MAX) && (next()->_beads[nl].prevOffset() != tt)) fprintf(stderr, "ERROR: next bead pl=%d prevOffset=%d != tt=%d\n", nl, next()->_beads[nl].prevOffset(), tt), nErrors++; if ((next() == NULL) && (nl != UINT16_MAX)) fprintf(stderr, "no next column, yet defined link for column %p bead %d\n", this, tt), nErrors++; } if (nErrors == 0) return; fprintf(stderr, "column %d %p has %d error in links\n", position(), this, nErrors); showLinks(); assert(nErrors == 0); }; void showLinks(uint16 ppos=UINT16_MAX, uint16 tpos=UINT16_MAX, uint16 npos=UINT16_MAX) { uint16 pmax = (_prevColumn == NULL) ? 0 : _prevColumn->_beadsLen; uint16 tmax = _beadsLen; uint16 nmax = (_nextColumn == NULL) ? 0 : _nextColumn->_beadsLen; uint16 lim = max(max(pmax, tmax), nmax); fprintf(stderr, "\n"); for (uint32 xx=0; xx_beads[xx].prevOffset() : UINT16_MAX, xx, (xx < pmax) ? _prevColumn->_beads[xx].nextOffset() : UINT16_MAX, (xx == ppos) ? '*' : ' ', (xx < tmax) ? _beads[xx].prevOffset() : UINT16_MAX, xx, (xx < tmax) ? _beads[xx].nextOffset() : UINT16_MAX, (xx == tpos) ? '*' : ' ', (xx < nmax) ? _nextColumn->_beads[xx].prevOffset() : UINT16_MAX, xx, (xx < nmax) ? _nextColumn->_beads[xx].nextOffset() : UINT16_MAX, (xx == npos) ? '*' : ' '); } }; friend class abAbacus; // friend bool mergeColumns(abColumn *lcolumn, abColumn *rcolumn); }; #endif // ABCOLUMN_H canu-1.3/src/utgcns/libcns/abMultiAlign.C000066400000000000000000000247071272335422300203400ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignment_CNS.C * src/AS_CNS/MultiAlignment_CNS.c * src/utgcns/libcns/MultiAlignment_CNS.C * * Modifications by: * * Michael Schatz on 2004-SEP-23 * are Copyright 2004 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2005-MAR-22 * are Copyright 2005 The Institute for Genomics Research, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2005-MAR-30 to 2008-FEB-13 * are Copyright 2005-2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Gennady Denisov from 2005-MAY-09 to 2008-JUN-06 * are Copyright 2005-2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUN-16 to 2013-AUG-01 * are Copyright 2005-2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Aaron Halpern from 2005-SEP-29 to 2006-OCT-03 * are Copyright 2005-2006 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-FEB-27 to 2009-MAY-14 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-AUG-11 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-01 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "abAbacus.H" void abAbacus::getConsensus(tgTig *tig) { // Resize the bases/quals storage to include a NUL byte. resizeArrayPair(tig->_gappedBases, tig->_gappedQuals, 0, tig->_gappedMax, _columnsLen + 1, resizeArray_doNothing); // Copy in the bases. memcpy(tig->_gappedBases, _cnsBases, sizeof(char) * _columnsLen); memcpy(tig->_gappedQuals, _cnsQuals, sizeof(uint8) * _columnsLen); // Terminate the strings. tig->_gappedBases[_columnsLen] = 0; tig->_gappedQuals[_columnsLen] = 0; // And set the length. tig->_gappedLen = _columnsLen; } // Crap. How can I figure out where a read starts in the multialign?? // Something needs to store the first column and index for the first bead....but // the column could be merged and removed. // // Put it in the column? A list of: // uint32 readId - the read id that starts here // uint16 link - the bead for that read // Adds one pointer to each column, most pointers are null. If the pointer is set, and we move bases, // we need to update. On a 32m unitig (pretty big) this adds 256mb + 1mb (100k reads at 8 bytes each) // // Put it in the read? If we move the first bead in the read, we know we need to update something, // but we have no link back to the read from the bead. // // It's so sparse - can we just hash it? Need to store: // readID -> column/link of first bead // column/link -> readID // The column probably needs to be a pointer (not the columnid) since the ID changes frequently. // Moving beads needs to update this structure, but any structure we make has the same problem. // index < length eliminates any endgaps from the delta list KAR, 09/19/02 uint32 abAbacus::getSequenceDeltas(uint32 sid, int32 *deltas) { abSequence *seq = getSequence(sid); beadID bid = readTofBead[sid]; abColumn *column = bid.column; uint16 link = bid.link; uint32 dl = 0; uint32 bp = 0; while (link != UINT16_MAX) { assert(column != NULL); char base = column->_beads[link].base(); if (base == '-') { if (deltas) deltas[dl] = bp; dl++; } else bp++; link = column->_beads[link].nextOffset(); column = column->next(); } if (deltas) deltas[dl] = 0; dl++; return(dl); } void abAbacus::getPositions(tgTig *tig) { uint32 nd = 0; for (uint32 si=0; si_childDeltas, tig->_childDeltasLen, tig->_childDeltasMax, nd, resizeArray_doNothing); tig->_childDeltasLen = 0; int32 maxPos = 0; for (uint32 si=0; sigetChild(si); // Assumes one-to-one map of seqs to children; assert(seq->gkpIdent() == child->ident()); beadID fBead = readTofBead[si]; beadID lBead = readTolBead[si]; if (fBead.column == NULL) { fprintf(stderr, "WARNING: read %u not in multialignment; position set to 0,0.\n", seq->gkpIdent()); child->setMinMax(0, 0); continue; } assert(fBead.column != NULL); assert(lBead.column != NULL); // Positions are zero-based and inclusive. The end position gets one added to it to make it true space-based. int32 min = fBead.column->position(); int32 max = lBead.column->position() + 1; if (maxPos < min) maxPos = min; if (maxPos < max) maxPos = max; if (seq->isRead() == true) { child->setMinMax(min, max); // The child remembers its orientation, and sets min/max appropriately. // Grab the deltas for real now, and set the deltaLen to one less than the number returned // (we don't care about the terminating zero!) child->_deltaOffset = tig->_childDeltasLen; child->_deltaLen = getSequenceDeltas(si, tig->_childDeltas + tig->_childDeltasLen); tig->_childDeltasLen += child->_deltaLen; child->_deltaLen--; //fprintf(stderr, "getPositions()-- si %u deltaLen %u offset %u\n", // si.get(), child->_deltaLen, child->_deltaOffset); } } // Hopefully, maxPos and _gappedLen agree. If not....? Note that positions are space-based if (maxPos != tig->_gappedLen) fprintf(stderr, "WARNING: maxPos=%d differs from gappedLen=%d. Huh?\n", maxPos, tig->_gappedLen); tig->_layoutLen = tig->_gappedLen; } // If called from unitigConsensus, this needs a rebuild() first. void abAbacus::display(FILE *F) { int32 pageWidth = 250; // For display, offset the quals to Sanger spec. for (uint32 ii=0; iigkpIdent(); fit[i].column = NULL; fit[i].link = UINT16_MAX; type[i] = (seq->isRead() == true) ? 'R' : '?'; bgn[i] = bgnColumn->position(); end[i] = endColumn->position(); } } fprintf(F,"\n==================== abAbacus::display ====================\n"); for (uint32 window_start=0; window_start < numberOfColumns(); ) { fprintf(F,"\n"); fprintf(F,"%d - %d (length %d)\n", window_start, window_start + pageWidth, numberOfColumns()); fprintf(F,"%-*.*s <<< consensus\n", pageWidth, pageWidth, _cnsBases + window_start); fprintf(F,"%-*.*s <<< quality\n\n", pageWidth, pageWidth, _cnsQuals + window_start); for (uint32 i=0; iposition() < wi) { fit[i].link = fit[i].column->_beads[fit[i].link].nextOffset(); fit[i].column = fit[i].column->next(); } } char pc = fit[i].column->_beads[fit[i].link].base(); if (pc == _cnsBases[wi]) pc = tolower(pc); else pc = toupper(pc); fprintf(F, "%c", pc); fit[i].link = fit[i].column->_beads[fit[i].link].nextOffset(); fit[i].column = fit[i].column->next(); } // Spaces after the read else if (end[i] < wi) { fprintf(F, ","); } // Sequence isn't involved in this window. else { fprintf(stderr, "bgn=%d end=%d wi=%d\n", bgn[i], end[i], wi); assert(0); } // If at the end of the window, print the id of the object we just printed. // if (wi == window_start + pageWidth - 1) fprintf(F," <<< %d (%c)\n", fid[i], type[i]); } } window_start += pageWidth; } // Unoffset the quals back to integers. for (uint32 ii=0; ii using namespace std; // Define this. Use the faster aligner from overlapper. If not defined, // a full O(n^2) DP is computed. // #undef WITH_NDALIGN #define WITH_NDALIGN unitigConsensus::unitigConsensus(gkStore *gkpStore_, double errorRate_, double errorRateMax_, uint32 minOverlap_) { gkpStore = gkpStore_; tig = NULL; numfrags = 0; trace = NULL; abacus = NULL; utgpos = NULL; cnspos = NULL; tiid = 0; piid = -1; minOverlap = minOverlap_; errorRate = errorRate_; errorRateMax = errorRateMax_; oaPartial = NULL; oaFull = NULL; } unitigConsensus::~unitigConsensus() { delete [] trace; delete abacus; delete [] utgpos; delete [] cnspos; delete oaPartial; delete oaFull; } void unitigConsensus::reportStartingWork(void) { if (showProgress()) fprintf(stderr, "unitigConsensus()-- processing read %u/%u id %d pos %d,%d anchor %d,%d,%d -- length %u\n", tiid+1, numfrags, utgpos[tiid].ident(), utgpos[tiid].min(), utgpos[tiid].max(), utgpos[tiid].anchor(), utgpos[tiid].aHang(), utgpos[tiid].bHang(), abacus->numberOfColumns()); if (showPlacementBefore()) for (int32 x=0; x<=tiid; x++) fprintf(stderr, "unitigConsensus()-- mid %10d utgpos %7d,%7d cnspos %7d,%7d anchor %10d,%6d,%6d\n", utgpos[x].ident(), utgpos[x].min(), utgpos[x].max(), cnspos[x].min(), cnspos[x].max(), utgpos[x].anchor(), utgpos[x].aHang(), utgpos[x].bHang()); } void unitigConsensus::reportFailure(void) { fprintf(stderr, "unitigConsensus()-- failed to align fragment %d in unitig %d.\n", utgpos[tiid].ident(), tig->tigID()); } void unitigConsensus::reportSuccess() { //fprintf(stderr, "unitigConsensus()-- fragment %d aligned in unitig %d.\n", // utgpos[tiid].ident(), tig->tigID()); } // Dump the unitig and reads to a single file. We should also probably save any parameters, // but then it's not clear what to do with them. // bool unitigConsensus::savePackage(FILE *outPackageFile, tgTig *tig) { // Saving the tig is easy, just use the standard dump. tig->saveToStream(outPackageFile); // Saving the reads is also easy, but it's a non-standard dump. for (uint32 ii=0; iinumberOfChildren(); ii++) gkpStore->gkStore_saveReadToStream(outPackageFile, tig->getChild(ii)->ident()); return(true); } bool unitigConsensus::generate(tgTig *tig_, map *inPackageRead_, map *inPackageReadData_) { tig = tig_; numfrags = tig->numberOfChildren(); if (initialize(inPackageRead_, inPackageReadData_) == FALSE) { fprintf(stderr, "generate()-- Failed to initialize for tig %u with %u children\n", tig->tigID(), tig->numberOfChildren()); goto returnFailure; } while (moreFragments()) { reportStartingWork(); // First attempt, all default parameters if (computePositionFromAnchor() && alignFragment()) goto applyAlignment; if (computePositionFromLayout() && alignFragment()) goto applyAlignment; if (computePositionFromAlignment() && alignFragment()) goto applyAlignment; // Second attempt, default parameters after recomputing consensus sequence. if (showAlgorithm()) fprintf(stderr, "generateMultiAlignment()-- recompute full consensus\n"); recomputeConsensus(showMultiAlignments()); if (computePositionFromAnchor() && alignFragment()) goto applyAlignment; if (computePositionFromLayout() && alignFragment()) goto applyAlignment; if (computePositionFromAlignment() && alignFragment()) goto applyAlignment; // Third attempot, use whatever aligns. (alignFragment(true) forced it to align, but that's breaking the consensus with garbage alignments) if (computePositionFromAlignment() && alignFragment(true)) goto applyAlignment; // Nope, failed to align. reportFailure(); continue; applyAlignment: setErrorRate(errorRate); setMinOverlap(minOverlap); reportSuccess(); abacus->applyAlignment(tiid, traceABgn, traceBBgn, trace, traceLen); refreshPositions(); } generateConsensus(tig); return(true); returnFailure: fprintf(stderr, "generateMultiAlignment()-- unitig %d FAILED.\n", tig->tigID()); // tgTig should have no changes. return(false); } bool unitigConsensus::generatePBDAG(tgTig *tig_, map *inPackageRead_, map *inPackageReadData_) { tig = tig_; numfrags = tig->numberOfChildren(); if (initialize(inPackageRead_, inPackageReadData_) == FALSE) { fprintf(stderr, "generatePBDAG()-- Failed to initialize for tig %u with %u children\n", tig->tigID(), tig->numberOfChildren()); return(false); } // first we need to load into Unitig data structure the quick cns Unitig utg; utg.id = tig->tigID(); utg.seq = string(tig->_layoutLen, 'N'); // build a quick consensus to align to, just smash together sequences. for (int i = 0; i < numfrags; i++) { gkRead *read = gkpStore->gkStore_getRead(utgpos[i].ident()); uint32 readLen = read->gkRead_sequenceLength(); uint32 start = utgpos[i].min(); uint32 end = utgpos[i].max(); if (start > utg.seq.length()) { start = utg.seq.length() - 1; } if (end - start > readLen) { end = start + readLen; } if (end > utg.seq.length()) { end = utg.seq.length() - 1; } abSequence *seq = abacus->getSequence(i); char *fragment = seq->getBases(); for (int j = start; j < end; j++) { if (utg.seq[j] == 'N') { utg.seq[j] = fragment[j - start]; } } } AlnGraphBoost ag(utg.seq); // compute alignments of each sequence in parallel #pragma omp parallel for schedule(dynamic) for (int i = 0; i < numfrags; i++) { bool placed = computePositionFromLayout(); dagcon::Alignment aln; SimpleAligner align; // for each fragment align it abSequence *seq = abacus->getSequence(i); char *fragment = seq->getBases(); aln.start = utgpos[i].min(); aln.end = utgpos[i].max(); aln.frgid = utgpos[i].ident(); aln.qstr = string(fragment); aln.tstr = utg.seq.substr(aln.start, aln.end-aln.start); align.align(aln, errorRate); if (aln.qstr.size() == 0) { cnspos[i].setMinMax(0, 0); continue; } cnspos[i].setMinMax(aln.start, aln.end); dagcon::Alignment norm = normalizeGaps(aln); // not thread safe to add to graph concurrently, so lock while adding #pragma omp critical (graphAdd) ag.addAln(norm); } // merge the nodes and call consensus ag.mergeNodes(); std::string cns = ag.consensus(1); // save consensus resizeArrayPair(tig->_gappedBases, tig->_gappedQuals, 0, tig->_gappedMax, (uint32) cns.length() + 1, resizeArray_doNothing); std::string::size_type len = 0; for(len = 0; len < cns.size(); len++) { tig->_gappedBases[len] = cns[len]; tig->_gappedQuals[len] = CNS_MIN_QV; } // Terminate the string. tig->_gappedBases[len] = 0; tig->_gappedQuals[len] = 0; tig->_gappedLen = len; tig->_layoutLen = len; assert(len < tig->_gappedMax); return true; } bool unitigConsensus::generateQuick(tgTig *tig_, map *inPackageRead_, map *inPackageReadData_) { tig = tig_; numfrags = tig->numberOfChildren(); if (initialize(inPackageRead_, inPackageReadData_) == FALSE) { fprintf(stderr, "generateMultiAlignment()-- Failed to initialize for tig %u with %u children\n", tig->tigID(), tig->numberOfChildren()); return(false); } // The quick variety doesn't generate alignments, it just pastes read bases into consensus. It // still needs to find a placement for the read, and it uses the other placed reads for that. while (moreFragments()) { reportStartingWork(); piid = -1; bool placed = computePositionFromLayout(); gkRead *read = gkpStore->gkStore_getRead(utgpos[tiid].ident()); uint32 readLen = read->gkRead_sequenceLength(); uint32 start = cnspos[tiid].min(); uint32 end = cnspos[tiid].max(); // if we couldn't place the read, fall back to utg positions if (placed == false) { start = abacus->numberOfColumns() - 1; end = start + readLen; } uint32 bHang = end - abacus->numberOfColumns(); if (bHang <= 0) { // this read doesn't add anything, skip it continue; } // check if our positions are wonky, adjust the end to match reality if (start > abacus->numberOfColumns()) { start = abacus->numberOfColumns(); } if (end - start > readLen) { end = start + readLen; } cnspos[tiid].setMinMax(start, end); // appendBases() will append only if new bases are needed. Otherwise, it just // sets the first/last bead position. abacus->appendBases(tiid, cnspos[tiid].min(), cnspos[tiid].max()); // I _think_ we need to rebuild iff bases are added. This also resets positions for each read. // Until someone complains this is too slow, it's left in. refreshPositions(); } generateConsensus(tig); return(true); } int unitigConsensus::initialize(map *inPackageRead, map *inPackageReadData) { int32 num_columns = 0; //int32 num_bases = 0; if (numfrags == 0) { fprintf(stderr, "utgCns::initialize()-- unitig has no children.\n"); return(false); } utgpos = new tgPosition [numfrags]; cnspos = new tgPosition [numfrags]; memcpy(utgpos, tig->getChild(0), sizeof(tgPosition) * numfrags); memcpy(cnspos, tig->getChild(0), sizeof(tgPosition) * numfrags); traceLen = 0; trace = new int32 [2 * AS_MAX_READLEN]; traceABgn = 0; traceBBgn = 0; memset(trace, 0, sizeof(int32) * 2 * AS_MAX_READLEN); abacus = new abAbacus(); // Clear the cnspos position. We use this to show it's been placed by consensus. // Guess the number of columns we'll end up with. // Initialize abacus with the reads. for (int32 i=0; i num_columns) ? utgpos[i].min() : num_columns; num_columns = (utgpos[i].max() > num_columns) ? utgpos[i].max() : num_columns; abacus->addRead(gkpStore, utgpos[i].ident(), utgpos[i]._askip, utgpos[i]._bskip, utgpos[i].isReverse(), inPackageRead, inPackageReadData); } // Check for duplicate reads { set dupFrag; for (uint32 i=0; itigID(), utgpos[i].ident()); return(false); } if (dupFrag.find(utgpos[i].ident()) != dupFrag.end()) { fprintf(stderr, "unitigConsensus()-- Unitig %d FAILED. Child %d is a duplicate.\n", tig->tigID(), utgpos[i].ident()); return(false); } dupFrag.insert(utgpos[i].ident()); } } // Initialize with the first read. abacus->applyAlignment(0, 0, 0, NULL, 0); // And set the placement of the first read. cnspos[0].setMinMax(0, abacus->numberOfColumns()); return(true); } int unitigConsensus::computePositionFromAnchor(void) { assert(piid == -1); uint32 anchor = utgpos[tiid].anchor(); if (anchor == 0) // No anchor?! Damn. goto computePositionFromAnchorFail; for (piid = tiid-1; piid >= 0; piid--) { abSequence *aseq = abacus->getSequence(piid); if (anchor != aseq->gkpIdent()) // Not the anchor. continue; if ((cnspos[piid].min() == 0) && (cnspos[piid].max() == 0)) // Is the anchor, but that isn't placed. goto computePositionFromAnchorFail; if ((utgpos[piid].max() < utgpos[tiid].min()) || (utgpos[tiid].max() < utgpos[piid].min())) { // Is the anchor, and anchor is placed, but the anchor doesn't agree with the placement. if (showPlacement()) fprintf(stderr, "computePositionFromAnchor()-- anchor %d at utg %d,%d doesn't agree with my utg %d,%d. FAIL\n", anchor, utgpos[piid].min(), utgpos[piid].max(), utgpos[tiid].min(), utgpos[tiid].max()); goto computePositionFromAnchorFail; } // Scale the hangs by the change in the anchor size between bogart and consensus. #if 0 double anchorScale = (double)(cnspos[piid].max() - cnspos[piid].min()) / (double)(utgpos[piid].max() - utgpos[piid].min()); if (showPlacement()) fprintf(stderr, "computePositionFromAnchor()-- frag %u in anchor %u -- hangs %d,%d -- scale %f -- final hangs %.0f,%.0f\n", utgpos[tiid].ident(), utgpos[piid].ident(), utgpos[tiid].aHang(), utgpos[tiid].bHang(), anchorScale, utgpos[tiid].aHang() * anchorScale, utgpos[tiid].bHang() * anchorScale); cnspos[tiid].setMinMax(cnspos[piid].min() + utgpos[tiid].aHang() * anchorScale, cnspos[piid].max() + utgpos[tiid].bHang() * anchorScale); // Hmmm, but if we shrank the read too much, add back in some of the length. We want to end up // with the read scaled by anchorScale, and centered on the hangs. int32 fragmentLength = utgpos[tiid].max() - utgpos[tiid].min(); if ((cnspos[tiid].min() >= cnspos[tiid].max()) || (cnspos[tiid].max() - cnspos[tiid].min() < 0.75 * fragmentLength)) { int32 center = (cnspos[tiid].min() + cnspos[tiid].max()) / 2; if (showPlacement()) { fprintf(stderr, "computePositionFromAnchor()-- frag %u in anchor %u -- too short. reposition around center %d with adjusted length %.0f\n", utgpos[tiid].ident(), utgpos[piid].ident(), center, fragmentLength * anchorScale); } cnspos[tiid].setMinMax(center - fragmentLength * anchorScale / 2, center + fragmentLength * anchorScale / 2); // We seem immune to having a negative position. We only use this to pull out a region from // the partial consensus to align to. // //if (cnspos[tiid].min() < 0) { // cnspos[tiid].min() = 0; // cnspos[tiid].max() = fragmentLength * anchorScale; //} } #else assert(0 <= utgpos[tiid].aHang()); uint32 bgn = abacus->getColumn(piid, cnspos[piid].min() + utgpos[tiid].aHang() - cnspos[piid].min()); uint32 end = abacus->getColumn(piid, cnspos[piid].max() + utgpos[tiid].bHang() - cnspos[piid].min()); cnspos[tiid].setMinMax(bgn, end); #endif assert(cnspos[tiid].min() < cnspos[tiid].max()); if (showPlacement()) fprintf(stderr, "computePositionFromAnchor()-- anchor %d at %d,%d --> beg,end %d,%d (tigLen %d)\n", anchor, cnspos[piid].min(), cnspos[piid].max(), cnspos[tiid].min(), cnspos[tiid].max(), abacus->numberOfColumns()); return(true); } computePositionFromAnchorFail: cnspos[tiid].setMinMax(0, 0); piid = -1; return(false); } int unitigConsensus::computePositionFromLayout(void) { int32 thickestLen = 0; assert(piid == -1); // Find the thickest qiid overlap to any cnspos fragment for (int32 qiid = tiid-1; qiid >= 0; qiid--) { if ((utgpos[tiid].min() < utgpos[qiid].max()) && (utgpos[tiid].max() > utgpos[qiid].min()) && ((cnspos[qiid].min() != 0) || (cnspos[qiid].max() != 0))) { cnspos[tiid].setMinMax(cnspos[qiid].min() + utgpos[tiid].min() - utgpos[qiid].min(), cnspos[qiid].max() + utgpos[tiid].max() - utgpos[qiid].max()); // This assert triggers. It results in 'ooo' below being negative, and we // discard this overlap anyway. // //assert(cnspos[tiid].min() < cnspos[tiid].max()); int32 ooo = MIN(cnspos[tiid].max(), abacus->numberOfColumns()) - cnspos[tiid].min(); #if 1 if (showPlacement()) fprintf(stderr, "computePositionFromLayout()-- layout %d at utg %d,%d cns %d,%d --> utg %d,%d cns %d,%d -- overlap %d\n", utgpos[qiid].ident(), utgpos[qiid].min(), utgpos[qiid].max(), cnspos[qiid].min(), cnspos[qiid].max(), utgpos[tiid].min(), utgpos[tiid].max(), cnspos[tiid].min(), cnspos[tiid].max(), ooo); #endif // Occasionally we see an overlap in the original placement (utgpos overlap) by after // adjusting our fragment to the consensus position, we no longer have an overlap. This // seems to be caused by a bad original placement. // // Example: // utgpos[a] = 13480,14239 cnspos[a] = 13622,14279 // utgpos[b] = 14180,15062 // // Our placement is 200bp different at the start, but close at the end. When we compute the // new start placement, it starts after the end of the A read -- the utgpos say the B read // starts 700bp after the A read, which is position 13622 + 700 = 14322....50bp after A ends. if ((cnspos[tiid].min() < abacus->numberOfColumns()) && (thickestLen < ooo)) { thickestLen = ooo; assert(cnspos[tiid].min() < cnspos[tiid].max()); // But we'll still assert cnspos is ordered correctly. int32 ovl = ooo; int32 ahang = cnspos[tiid].min(); int32 bhang = cnspos[tiid].max() - abacus->numberOfColumns(); piid = qiid; } } } // If we have a VALID thickest placement, use that (recompute the placement that is likely // overwritten -- ahang, bhang and piid are still correct). if (thickestLen >= minOverlap) { assert(piid != -1); cnspos[tiid].setMinMax(cnspos[piid].min() + utgpos[tiid].min() - utgpos[piid].min(), cnspos[piid].max() + utgpos[tiid].max() - utgpos[piid].max()); assert(cnspos[tiid].min() < cnspos[tiid].max()); if (showPlacement()) fprintf(stderr, "computePositionFromLayout()-- layout %d at %d,%d --> beg,end %d,%d (tigLen %d)\n", utgpos[piid].ident(), cnspos[piid].min(), cnspos[piid].max(), cnspos[tiid].min(), cnspos[tiid].max(), abacus->numberOfColumns()); return(true); } cnspos[tiid].setMinMax(0, 0); piid = -1; return(false); } // Occasionally we get a fragment that just refuses to go in the correct spot. Search for the // correct placement in all of consensus, update ahang,bhang and retry. // // We don't expect to have big negative ahangs, and so we don't allow them. To unlimit this, use // "-fragmentLen" instead of the arbitrary cutoff below. int unitigConsensus::computePositionFromAlignment(void) { assert(piid == -1); int32 minlen = minOverlap; int32 ahanglimit = -10; abSequence *seq = abacus->getSequence(tiid); char *fragment = seq->getBases(); int32 fragmentLen = seq->length(); bool foundAlign = false; // // Try NDalign. // if (foundAlign == false) { if (oaPartial == false) oaPartial = new NDalign(pedLocal, errorRate, 17); // partial allowed! oaPartial->initialize(0, abacus->bases(), abacus->numberOfColumns(), 0, abacus->numberOfColumns(), 1, fragment, fragmentLen, 0, fragmentLen, false); if ((oaPartial->findMinMaxDiagonal(minOverlap) == true) && (oaPartial->findSeeds(false) == true) && (oaPartial->findHits() == true) && (oaPartial->chainHits() == true) && (oaPartial->processHits() == true)) { cnspos[tiid].setMinMax(oaPartial->abgn(), oaPartial->aend()); //fprintf(stderr, "computePositionFromAlignment()-- cnspos[%3d] mid %d %d,%d (from NDalign)\n", tiid, utgpos[tiid].ident(), cnspos[tiid].min(), cnspos[tiid].max()); foundAlign = true; } } // // Fail. // if (foundAlign == false) { cnspos[tiid].setMinMax(0, 0); piid = -1; if (showAlgorithm()) fprintf(stderr, "computePositionFromAlignment()-- Returns fail (no alignment).\n"); return(false); } // From the overlap and existing placements, find the thickest overlap, to set the piid and // hangs, then reset the original placement based on that anchors original placement. // // To work with fixFailures(), we need to scan the entire fragment list. This isn't so bad, // really, since before we were scanning (on average) half of it. assert(cnspos[tiid].min() < cnspos[tiid].max()); int32 thickestLen = 0; for (int32 qiid = numfrags-1; qiid >= 0; qiid--) { if ((tiid != qiid) && (cnspos[tiid].min() < cnspos[qiid].max()) && (cnspos[tiid].max() > cnspos[qiid].min())) { int32 ooo = (MIN(cnspos[tiid].max(), cnspos[qiid].max()) - MAX(cnspos[tiid].min(), cnspos[qiid].min())); if (thickestLen < ooo) { thickestLen = ooo; int32 ovl = ooo; int32 ahang = cnspos[tiid].min(); int32 bhang = cnspos[tiid].max() - abacus->numberOfColumns(); piid = qiid; } } } // No thickest? Dang. if (thickestLen == 0) { cnspos[tiid].setMinMax(0, 0); piid = -1; if (showAlgorithm()) fprintf(stderr, "computePositionFromAlignment()-- Returns fail (no thickest).\n"); return(false); } // Success, yay! assert(piid != -1); if (showPlacement()) fprintf(stderr, "computePositionFromAlignment()-- layout %d at %d,%d --> beg,end %d,%d (tigLen %d)\n", utgpos[piid].ident(), cnspos[piid].min(), cnspos[piid].max(), cnspos[tiid].min(), cnspos[tiid].max(), abacus->numberOfColumns()); return(true); } void unitigConsensus::generateConsensus(tgTig *tig) { abacus->recallBases(true); // Do one last base call, using the full works. abacus->refine(abAbacus_Smooth); abacus->mergeColumns(true); abacus->refine(abAbacus_Poly_X); abacus->mergeColumns(true); abacus->refine(abAbacus_Indel); abacus->mergeColumns(true); abacus->recallBases(true); // The bases are possibly all recalled, depending on the above refinements keeping things consistent. //abacus->refreshColumns(); // Definitely needed, this copies base calls into _cnsBases and _cnsQuals. // Copy the consensus and positions into the tig. abacus->getConsensus(tig); abacus->getPositions(tig); // While we have fragments in memory, compute the microhet probability. Ideally, this would be // done in CGW when loading unitigs (the only place the probability is used) but the code wants // to load sequence and quality for every fragment, and that's too expensive. } // Update the position of each fragment in the consensus sequence. // Update the anchor/hang of the fragment we just placed. void unitigConsensus::refreshPositions(void) { for (int32 i=0; i<=tiid; i++) { if ((cnspos[i].min() == 0) && (cnspos[i].max() == 0)) // Uh oh, not placed originally. continue; abColumn *fcol = abacus->readTofBead[i].column; abColumn *lcol = abacus->readTolBead[i].column; cnspos[i].setMinMax(fcol->position(), lcol->position() + 1); assert(cnspos[i].min() >= 0); assert(cnspos[i].max() > cnspos[i].min()); } if (piid >= 0) utgpos[tiid].setAnchor(utgpos[piid].ident(), cnspos[tiid].min() - cnspos[piid].min(), cnspos[tiid].max() - cnspos[piid].max()); piid = -1; } // Run abacus to rebuild the consensus sequence. VERY expensive. void unitigConsensus::recomputeConsensus(bool display) { //abacus->recallBases(false); // Needed? We should be up to date. abacus->refine(abAbacus_Smooth); abacus->mergeColumns(false); abacus->refine(abAbacus_Poly_X); abacus->mergeColumns(false); abacus->refine(abAbacus_Indel); abacus->mergeColumns(false); abacus->recallBases(false); // Possibly not needed. If this is removed, the following refresh is definitely needed. //abacus->refreshColumns(); // Definitely needed, this copies base calls into _cnsBases and _cnsQuals. refreshPositions(); if (display) abacus->display(stderr); } // This stub lets alignFragmnet() cleanup and return on alignment failures. The original // implementation did the same thing with a goto to the end of the function. Opening up the if // statements exposed variable declarations that prevented the goto from compiling. // bool unitigConsensus::alignFragmentFailure(void) { cnspos[tiid].setMinMax(0, 0); piid = -1; if (showAlgorithm()) fprintf(stderr, "alignFragment()-- No alignment found.\n"); return(false); } // Generates an alignment of the current read to the partial consensus. // The primary output is a trace stored in the object data. bool unitigConsensus::alignFragment(bool forceAlignment) { assert((cnspos[tiid].min() != 0) || (cnspos[tiid].max() != 0)); assert(piid != -1); assert(cnspos[tiid].min() < cnspos[tiid].max()); abSequence *bSEQ = abacus->getSequence(tiid); char *fragSeq = bSEQ->getBases(); int32 fragLen = bSEQ->length(); // Decide on how much to align. Pick too little of consensus, and we leave some of the read // unaligned. Pick too much, and the read aligns poorly. // // endTrim is trimmed from the 3' of the read. This is the stuff we don't expect to align to consensus. // bgnExtra is trimmed from the 5' of consensus. Same idea as endTrim. // endExtra is trimmed from the 3' of consensus. Only for contained reads. // // These values are adjusted later, in trimStep increments, based on the alignments returned. // // Of the two choices, making the two Extra's small at the start is probably safer. That case is // easy to detect, and easy to fix. // // The expectedAlignLen is almost always an underestimate. Any gaps inserted will make the real // alignment length longer. This used to be multiplied by the error rate. int32 expectedAlignLen = cnspos[tiid].max() - cnspos[tiid].min(); // If the read is contained, the full read is aligned. // Otherwise, an extra 1/32 of the align length is added for padding. int32 fragBgn = 0; int32 fragEnd = (cnspos[tiid].max() < abacus->numberOfColumns()) ? (fragLen) : (33 * expectedAlignLen / 32); if (fragEnd > fragLen) fragEnd = fragLen; // Given the usual case of using an actual overlap to a read in the multialign to find the region // to align to, we expect that region to be nearly perfect. Thus, we shouldn't need to extend it // much. If anything, we'll need to extend the 3' end of the read. int32 bgnExtra = 10; // Start with a small 'extra' allowance, easy to make bigger. int32 endExtra = 10; // int32 trimStep = max(10, expectedAlignLen / 50); // Step by 10 bases or do at most 50 steps. // Find an alignment! bool allowedToTrim = true; assert(abacus->bases()[abacus->numberOfColumns()] == 0); // Consensus must be NUL terminated assert(fragSeq[fragLen] == 0); // The read must be NUL terminated alignFragmentAgain: // Truncate consensus and the read to prevent false alignments. if (cnspos[tiid].max() + endExtra > abacus->numberOfColumns()) endExtra = abacus->numberOfColumns() - cnspos[tiid].max(); int32 cnsBgn = MAX(0, cnspos[tiid].min() - bgnExtra); // Start position in consensus int32 cnsEnd = cnspos[tiid].max() + endExtra; // Truncation of consensus int32 cnsEndBase = abacus->bases()[cnsEnd]; // Saved base (if not truncated, it's the NUL byte at the end) char *aseq = abacus->bases() + cnsBgn; char *bseq = fragSeq; char fragEndBase = bseq[fragEnd]; // Saved base abacus->bases()[cnsEnd] = 0; // Do the truncations. bseq[fragEnd] = 0; // Report! if (showAlgorithm()) fprintf(stderr, "alignFragment()-- Allow bgnExtra=%d and endExtra=%d (cnsBgn=%d cnsEnd=%d cnsLen=%d) (fragBgn=0 fragEnd=%d fragLen=%d)\n", bgnExtra, endExtra, cnsBgn, cnsEnd, abacus->numberOfColumns(), fragEnd, fragLen); // Create new aligner object. 'Global' in this case just means to not stop early, not a true global alignment. if (oaFull == false) oaFull = new NDalign(pedGlobal, errorRate, 17); oaFull->initialize(0, aseq, cnsEnd - cnsBgn, 0, cnsEnd - cnsBgn, 1, bseq, fragEnd - fragBgn, 0, fragEnd - fragBgn, false); // Generate a null hit, then align it and then realign, from both endpoints, and save the better // of the two. if ((oaFull->makeNullHit() == true) && (oaFull->processHits() == true)) { if (showAlignments()) oaFull->display("utgCns::alignFragment()--", true); oaFull->realignBackward(showAlgorithm(), showAlignments()); oaFull->realignForward (showAlgorithm(), showAlignments()); } // Restore the bases we removed to end the strings early. if (cnsEndBase) abacus->bases()[cnsEnd] = cnsEndBase; if (fragEndBase) bseq[fragEnd] = fragEndBase; // If no alignment, bail. if (oaFull->length() == 0) return(alignFragmentFailure()); // // Check quality and fail if it sucks. // bool isBad = oaFull->scanDeltaForBadness(showAlgorithm(), showAlignments()); // Check for bad (under) trimming of input sequences. // // If the alignment is bad, and we hit the start of the consensus sequence (or the end of // same), chances are good that the aligner returned a (higher scoring) global alignment instead // of a (lower scoring) local alignment. Trim off some of the extension and try again. if ((allowedToTrim == true) && (isBad == true) && (oaFull->ahg5() == 0) && (bgnExtra > 0)) { int32 adj = (bgnExtra < trimStep) ? 0 : bgnExtra - trimStep; if (showAlgorithm()) fprintf(stderr, "utgCns::alignFragment()-- alignment is bad, hit the trimmed start of consensus, decrease bgnExtra from %u to %u\n", bgnExtra, adj); bgnExtra = adj; goto alignFragmentAgain; } if ((allowedToTrim == true) && (isBad == true) && (oaFull->ahg3() == 0) && (endExtra > 0)) { int32 adj = (endExtra < trimStep) ? 0 : endExtra - trimStep; if (showAlgorithm()) fprintf(stderr, "utgCns::alignFragment()-- alignment is bad, hit the trimmed end of consensus, decrease endExtra from %u to %u\n", endExtra, adj); endExtra = adj; goto alignFragmentAgain; } // Check for bad (over) trimming of input sequences. Bad if: // we don't hit the start of the read, and we chopped the start of consensus. // we don't hit the end of the read, and we chopped the end of consensus. // we do hit the end of the read, and we chopped the end of the read. // (we don't chop the start of the read, so the fourth possible case never happens) allowedToTrim = false; // No longer allowed to reduce bgnExtra or endExtra. We'd hit infinite loops otherwise. if ((oaFull->bhg5() > 0) && (cnsBgn > 0)) { int32 adj = bgnExtra + 2 * oaFull->bhg5(); if (showAlgorithm()) fprintf(stderr, "utgCns::alignFragment()-- hit the trimmed start of consensus, increase bgnExtra from %u to %u\n", bgnExtra, adj); bgnExtra = adj; goto alignFragmentAgain; } if ((oaFull->bhg3() > 0) && (cnsEnd < abacus->numberOfColumns())) { int32 adj = endExtra + 2 * oaFull->bhg3(); if (showAlgorithm()) fprintf(stderr, "utgCns::alignFragment()-- hit the trimmed end of consensus, increase endExtra from %u to %u\n", endExtra, adj); endExtra = adj; goto alignFragmentAgain; } if ((oaFull->bhg3() == 0) && (fragEnd < fragLen)) { int32 adj = (fragEnd + trimStep < fragLen) ? fragEnd + trimStep : fragLen; if (showAlgorithm()) fprintf(stderr, "utgCns::alignFragment()-- hit the trimmed end of the read, increase fragEnd from %d to %d\n", fragEnd, adj); fragEnd = adj; goto alignFragmentAgain; } // If we get here, and it's still bad, well, not much we can do. if ((forceAlignment == false) && (isBad == true)) { if (showAlgorithm()) fprintf(stderr, "utgCns::alignFragment()-- alignment bad after realigning\n"); return(alignFragmentFailure()); } if ((forceAlignment == false) && (oaFull->erate() > errorRate)) { if (showAlgorithm()) { fprintf(stderr, "utgCns::alignFragment()-- alignment is low quality: %f > %f\n", oaFull->erate(), errorRate); oaFull->display("utgCns::alignFragment()-- ", true); } return(alignFragmentFailure()); } // Otherwise, its a good alignment. Process the trace to the 'consensus-format' and return true. // // Set the begin points of the trace. We probably need at least one of abgn and bbgn to be // zero. If both are nonzero, then we have a branch in the alignment. // // If traceABgn is negative, we insert gaps into A before it starts, but I'm not sure how that works. // // Overlap encoding: // Add N matches or mismatches. If negative, insert a base in the first sequence. If // positive, delete a base in the first sequence. // // Consensus encoding: // If negative, align (-trace - apos) bases, then add a gap in A. // If positive, align ( trace - bpos) bases, then add a gap in B. // if (oaFull->abgn() > 0) assert(oaFull->bbgn() == 0); // read aligned fully if consensus isn't if (oaFull->bbgn() > 0) assert(oaFull->abgn() == 0); // read extends past the begin, consensus aligned fully if (oaFull->bbgn() > 0) assert(cnsBgn == 0); // read extends past the begin, consensus not trimmed at begin traceABgn = cnsBgn + oaFull->abgn() - oaFull->bbgn(); traceBBgn = oaFull->bbgn(); int32 apos = oaFull->abgn(); int32 bpos = oaFull->bbgn(); traceLen = 0; for (uint32 ii=0; iideltaLen(); ii++, traceLen++) { if (oaFull->delta()[ii] < 0) { apos += -oaFull->delta()[ii] - 1; bpos += -oaFull->delta()[ii]; trace[traceLen] = -apos - cnsBgn - 1; } else { apos += oaFull->delta()[ii]; bpos += oaFull->delta()[ii] - 1; // critical trace[traceLen] = bpos + 1; } } trace[traceLen] = 0; return(true); } canu-1.3/src/utgcns/libcns/unitigConsensus.H000066400000000000000000000134301272335422300211640ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/MultiAlignment_CNS.h * src/AS_CNS/MultiAlignment_CNS_private.H * src/AS_CNS/MultiAlignment_CNS_private.h * src/utgcns/libcns/MultiAlignment_CNS_private.H * * Modifications by: * * Gennady Denisov from 2005-MAY-23 to 2007-OCT-25 * are Copyright 2005-2007 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2005-JUL-08 to 2013-AUG-01 * are Copyright 2005-2009,2011,2013 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Eli Venter from 2006-FEB-13 to 2008-FEB-13 * are Copyright 2006,2008 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Sergey Koren from 2008-JAN-28 to 2009-SEP-25 * are Copyright 2008-2009 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Jason Miller on 2011-SEP-21 * are Copyright 2011 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-NOV-17 to 2015-AUG-05 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-09 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2015-DEC-28 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef UNITIGCONSENSUS_H #define UNITIGCONSENSUS_H #include "AS_global.H" #include "tgStore.H" #include "abAbacus.H" class ALNoverlap; class NDalign; class unitigConsensus { public: unitigConsensus(gkStore *gkpStore_, double errorRate_, double errorRateMax_, uint32 minOverlap_); ~unitigConsensus(); bool savePackage(FILE *outPackageFile, tgTig *tig); bool generate(tgTig *tig, map *inPackageRead = NULL, map *inPackageReadData = NULL); bool generatePBDAG(tgTig *tig, map *inPackageRead = NULL, map *inPackageReadData = NULL); bool generateQuick(tgTig *tig, map *inPackageRead = NULL, map *inPackageReadData = NULL); int32 initialize(map *inPackageRead, map *inPackageReadData); void setErrorRate(double errorRate_) { errorRate = errorRate_; }; void setMinOverlap(uint32 minOverlap_) { minOverlap = minOverlap_; }; bool showProgress(void) { return(tig->_utgcns_verboseLevel >= 1); }; // -V displays which reads are processing bool showAlgorithm(void) { return(tig->_utgcns_verboseLevel >= 2); }; // -V -V displays some details on the algorithm bool showPlacementBefore(void) { return(tig->_utgcns_verboseLevel >= 3); }; // -V -V -V displays placement info before each read bool showPlacement(void) { return(tig->_utgcns_verboseLevel >= 3); }; // bool showAlignments(void) { return(tig->_utgcns_verboseLevel >= 4); }; // -V -V -V -V displays aligns and multialigns bool showMultiAlignments(void) { return(tig->_utgcns_verboseLevel >= 4); }; // void reportStartingWork(void); void reportFailure(void); void reportSuccess(void); int32 moreFragments(void) { tiid++; return (tiid < numfrags); }; int32 computePositionFromAnchor(void); int32 computePositionFromLayout(void); int32 computePositionFromAlignment(void); void recomputeConsensus(bool display); void refreshPositions(void); bool rejectAlignment(bool allowBhang, bool allowAhang, ALNoverlap *O); bool alignFragmentFailure(void); bool alignFragment(bool forceAlignment=false); void generateConsensus(tgTig *tig); private: gkStore *gkpStore; tgTig *tig; uint32 numfrags; // == tig->numberOfChildren() uint32 traceLen; uint32 traceMax; int32 *trace; int32 traceABgn; // Start of the aligned regions, discovered in alignFragment(), int32 traceBBgn; // used in applyAlignment(). abAbacus *abacus; // The two positions below are storing the low/high coords for the read. // They do not encode the orientation in the coordinates. // tgPosition *utgpos; // Original unitigger location. tgPosition *cnspos; // Actual location in frankenstein. int32 tiid; // This frag IID int32 piid; // Anchor frag IID - if -1, not valid uint32 minOverlap; double errorRate; double errorRateMax; NDalign *oaPartial; NDalign *oaFull; }; #endif canu-1.3/src/utgcns/libpbutgcns/000077500000000000000000000000001272335422300167145ustar00rootroot00000000000000canu-1.3/src/utgcns/libpbutgcns/.gitignore000066400000000000000000000003031272335422300207000ustar00rootroot00000000000000# Compiled Object files and Bindaries *.slo *.lo *.o pbutgcns # Compiled Dynamic libraries *.so *.dylib # Compiled Static libraries *.lai *.la *.a # dependency files *.d # vim tmpfiles *.swp canu-1.3/src/utgcns/libpbutgcns/Alignment.C000066400000000000000000000057411272335422300207450ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2015-DEC-28 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include #include #include #include #include "Alignment.H" using namespace dagcon; Alignment::Alignment() : start(0), end(0), frgid(""), qstr(""), tstr("") { } Alignment normalizeGaps(Alignment& aln) { size_t qlen = aln.qstr.length(), tlen = aln.tstr.length(); assert(qlen == tlen); std::string qNorm, tNorm; // convert mismatches to indels for (size_t i=0; i < qlen; i++) { char qb = aln.qstr[i], tb = aln.tstr[i]; if (qb != tb && qb != '-' && tb != '-') { qNorm += '-'; qNorm += qb; tNorm += tb; tNorm += '-'; } else { qNorm += qb; tNorm += tb; } } // update lengths qlen = qNorm.length(); tlen = tNorm.length(); // push gaps to the right, but not past the end for (size_t i=0; i < qlen-1; i++) { // pushing target gaps if (tNorm[i] == '-') { size_t j = i; while (true) { char c = tNorm[++j]; if (c != '-' || j > qlen - 1) { if (c == qNorm[i]) { tNorm[i] = c; tNorm[j] = '-'; } break; } } } // pushing query gaps if (qNorm[i] == '-') { size_t j = i; while (true) { char c = qNorm[++j]; if (c != '-' || j > tlen - 1) { if (c == tNorm[i]) { qNorm[i] = c; qNorm[j] = '-'; } break; } } } } // generate the final, normalized alignment strings Alignment finalNorm; finalNorm.frgid = aln.frgid; finalNorm.start = aln.start; for (size_t i=0; i < qlen; i++) { if (qNorm[i] != '-' || tNorm[i] != '-') { finalNorm.qstr += qNorm[i]; finalNorm.tstr += tNorm[i]; } } return finalNorm; } canu-1.3/src/utgcns/libpbutgcns/Alignment.H000066400000000000000000000045721272335422300207530ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2015-DEC-28 * are a 'United States Government Work', and * are released in the public domain * * Brian P. Walenz beginning on 2016-JAN-04 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef __GCON_ALIGNMENT_HPP__ #define __GCON_ALIGNMENT_HPP__ /// /// Super-simple alignment representation. Represents an alignment between two /// PacBio reads, one of which we're trying to correct. The read to correct /// may be either the target or the query, depending on how the alignment was /// done. /// #include #include namespace dagcon { class Alignment { public: // conforming offsets are 1-based uint32_t start; uint32_t end; // Fragment ID std::string frgid; // query and target strings must be equal length std::string qstr; std::string tstr; Alignment(); }; } struct Unitig { uint32_t id; std::string seq; }; /// Simplifies the alignment by normalizing gaps. Converts mismatches into /// indels ... /// query: CAC query: C-AC /// | | ---> | | /// target: CGC target: CG-C /// /// Shifts equivalent gaps to the right in the reference ... /// query: CAACAT query: CAACAT /// | | || ---> ||| | /// target: C-A-AT target: CAA--T /// /// Shifts equivalent gaps to the right in the read ... /// query: -C--CGT query: CCG--T /// | | | ---> ||| | /// target: CCGAC-T target: CCGACT dagcon::Alignment normalizeGaps(dagcon::Alignment& aln); #endif // __GCON_ALIGNMENT_HPP__ canu-1.3/src/utgcns/libpbutgcns/AlnGraphBoost.C000066400000000000000000000423721272335422300215330ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2015-DEC-28 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ // Copyright (c) 2011-2015, Pacific Biosciences of California, Inc. // // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted (subject to the limitations in the // disclaimer below) provided that the following conditions are met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following // disclaimer in the documentation and/or other materials provided // with the distribution. // // * Neither the name of Pacific Biosciences nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE // GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC // BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF // USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT // OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF // SUCH DAMAGE. #include #include #include #include #include #include #include #include #include #include #include "Alignment.H" #include "AlnGraphBoost.H" AlnGraphBoost::AlnGraphBoost(const std::string& backbone) { // initialize the graph structure with the backbone length + enter/exit // vertex size_t blen = backbone.length(); _g = G(blen+1); for (size_t i = 0; i < blen+1; i++) boost::add_edge(i, i+1, _g); VtxIter curr, last; boost::tie(curr, last) = boost::vertices(_g); _enterVtx = *curr++; _g[_enterVtx].base = '^'; _g[_enterVtx].backbone = true; for (size_t i = 0; i < blen; i++, ++curr) { VtxDesc v = *curr; _g[v].backbone = true; _g[v].weight = 1; _g[v].base = backbone[i]; _bbMap[v] = v; } _exitVtx = *curr; _g[_exitVtx].base = '$'; _g[_exitVtx].backbone = true; } AlnGraphBoost::AlnGraphBoost(const size_t blen) { _g = G(blen+1); for (size_t i = 0; i < blen+1; i++) boost::add_edge(i, i+1, _g); VtxIter curr, last; boost::tie(curr, last) = boost::vertices(_g); _enterVtx = *curr++; _g[_enterVtx].base = '^'; _g[_enterVtx].backbone = true; for (size_t i = 0; i < blen; i++, ++curr) { VtxDesc v = *curr; _g[v].backbone = true; _g[v].weight = 1; _g[v].deleted = false; _g[v].base = 'N'; _bbMap[v] = v; } _exitVtx = *curr; _g[_exitVtx].base = '$'; _g[_exitVtx].backbone = true; } void AlnGraphBoost::addAln(dagcon::Alignment& aln) { IndexMap index = boost::get(boost::vertex_index, _g); // tracks the position on the backbone uint32_t bbPos = aln.start; VtxDesc prevVtx = _enterVtx; for (size_t i = 0; i < aln.qstr.length(); i++) { char queryBase = aln.qstr[i], targetBase = aln.tstr[i]; VtxDesc currVtx = index[bbPos]; // match if (queryBase == targetBase) { _g[_bbMap[currVtx]].coverage++; // NOTE: for empty backbones _g[_bbMap[currVtx]].base = targetBase; _g[currVtx].weight++; addEdge(prevVtx, currVtx); bbPos++; prevVtx = currVtx; // query deletion } else if (queryBase == '-' && targetBase != '-') { _g[_bbMap[currVtx]].coverage++; // NOTE: for empty backbones _g[_bbMap[currVtx]].base = targetBase; bbPos++; // query insertion } else if (queryBase != '-' && targetBase == '-') { // create new node and edge VtxDesc newVtx = boost::add_vertex(_g); _g[newVtx].base = queryBase; _g[newVtx].weight++; _g[newVtx].backbone = false; _g[newVtx].deleted = false; _bbMap[newVtx] = bbPos; addEdge(prevVtx, newVtx); prevVtx = newVtx; } } addEdge(prevVtx, _exitVtx); } void AlnGraphBoost::addEdge(VtxDesc u, VtxDesc v) { // Check if edge exists with prev node. If it does, increment edge counter, // otherwise add a new edge. InEdgeIter ii, ie; bool edgeExists = false; for (boost::tie(ii, ie) = boost::in_edges(v, _g); ii != ie; ++ii) { EdgeDesc e = *ii; if (boost::source(e , _g) == u) { // increment edge count _g[e].count++; edgeExists = true; } } if (! edgeExists) { // add new edge std::pair p = boost::add_edge(u, v, _g); _g[p.first].count++; } } void AlnGraphBoost::mergeNodes() { std::queue seedNodes; seedNodes.push(_enterVtx); while(true) { if (seedNodes.size() == 0) break; VtxDesc u = seedNodes.front(); seedNodes.pop(); mergeInNodes(u); mergeOutNodes(u); OutEdgeIter oi, oe; for (boost::tie(oi, oe) = boost::out_edges(u, _g); oi != oe; ++oi) { EdgeDesc e = *oi; _g[e].visited = true; VtxDesc v = boost::target(e, _g); InEdgeIter ii, ie; int notVisited = 0; for (boost::tie(ii, ie) = boost::in_edges(v, _g); ii != ie; ++ii) { if (_g[*ii].visited == false) notVisited++; } // move onto the boost::target node after we visit all incoming edges for // the boost::target node if (notVisited == 0) seedNodes.push(v); } } } void AlnGraphBoost::mergeInNodes(VtxDesc n) { std::map > nodeGroups; InEdgeIter ii, ie; // Group neighboring nodes by base for(boost::tie(ii, ie) = boost::in_edges(n, _g); ii != ie; ++ii) { VtxDesc inNode = boost::source(*ii, _g); if (out_degree(inNode, _g) == 1) { nodeGroups[_g[inNode].base].push_back(inNode); } } // iterate over node groups, merge an accumulate information for(std::map >::iterator kvp = nodeGroups.begin(); kvp != nodeGroups.end(); ++kvp) { std::vector nodes = (*kvp).second; if (nodes.size() <= 1) continue; std::vector::const_iterator ni = nodes.begin(); VtxDesc an = *ni++; OutEdgeIter anoi, anoe; boost::tie(anoi, anoe) = boost::out_edges(an, _g); // Accumulate out edge information for (; ni != nodes.end(); ++ni) { OutEdgeIter oi, oe; boost::tie(oi, oe) = boost::out_edges(*ni, _g); _g[*anoi].count += _g[*oi].count; _g[an].weight += _g[*ni].weight; } // Accumulate in edge information, merges nodes ni = nodes.begin(); ++ni; for (; ni != nodes.end(); ++ni) { InEdgeIter ii, ie; VtxDesc n = *ni; for (boost::tie(ii, ie) = boost::in_edges(n, _g); ii != ie; ++ii) { VtxDesc n1 = boost::source(*ii, _g); EdgeDesc e; bool exists; boost::tie(e, exists) = edge(n1, an, _g); if (exists) { _g[e].count += _g[*ii].count; } else { std::pair p = boost::add_edge(n1, an, _g); _g[p.first].count = _g[*ii].count; _g[p.first].visited = _g[*ii].visited; } } markForReaper(n); } mergeInNodes(an); } } void AlnGraphBoost::mergeOutNodes(VtxDesc n) { std::map > nodeGroups; OutEdgeIter oi, oe; for(boost::tie(oi, oe) = boost::out_edges(n, _g); oi != oe; ++oi) { VtxDesc outNode = boost::target(*oi, _g); if (in_degree(outNode, _g) == 1) { nodeGroups[_g[outNode].base].push_back(outNode); } } for(std::map >::iterator kvp = nodeGroups.begin(); kvp != nodeGroups.end(); ++kvp) { std::vector nodes = (*kvp).second; if (nodes.size() <= 1) continue; std::vector::const_iterator ni = nodes.begin(); VtxDesc an = *ni++; InEdgeIter anii, anie; boost::tie(anii, anie) = boost::in_edges(an, _g); // Accumulate inner edge information for (; ni != nodes.end(); ++ni) { InEdgeIter ii, ie; boost::tie(ii, ie) = boost::in_edges(*ni, _g); _g[*anii].count += _g[*ii].count; _g[an].weight += _g[*ni].weight; } // Accumulate and merge outer edge information ni = nodes.begin(); ++ni; for (; ni != nodes.end(); ++ni) { OutEdgeIter oi, oe; VtxDesc n = *ni; for (boost::tie(oi, oe) = boost::out_edges(n, _g); oi != oe; ++oi) { VtxDesc n2 = boost::target(*oi, _g); EdgeDesc e; bool exists; boost::tie(e, exists) = edge(an, n2, _g); if (exists) { _g[e].count += _g[*oi].count; } else { std::pair p = boost::add_edge(an, n2, _g); _g[p.first].count = _g[*oi].count; _g[p.first].visited = _g[*oi].visited; } } markForReaper(n); } } } void AlnGraphBoost::markForReaper(VtxDesc n) { _g[n].deleted = true; clear_vertex(n, _g); _reaperBag.push_back(n); } void AlnGraphBoost::reapNodes() { int reapCount = 0; std::sort(_reaperBag.begin(), _reaperBag.end()); std::vector::iterator curr = _reaperBag.begin(); for (; curr != _reaperBag.end(); ++curr) { assert(_g[*curr].backbone==false); remove_vertex(*curr-reapCount++, _g); } } const std::string AlnGraphBoost::consensus(int minWeight) { // get the best scoring path std::vector path = bestPath(); // consensus sequence std::string cns; // track the longest consensus path meeting minimum weight int offs = 0, bestOffs = 0, length = 0, idx = 0; bool metWeight = false; std::vector::iterator curr = path.begin(); for (; curr != path.end(); ++curr) { AlnNode n = *curr; if (n.base == _g[_enterVtx].base || n.base == _g[_exitVtx].base) continue; cns += n.base; // initial beginning of minimum weight section if (!metWeight && n.weight >= minWeight) { offs = idx; metWeight = true; } else if (metWeight && n.weight < minWeight) { // concluded minimum weight section, update if longest seen so far if ((idx - offs) > length) { bestOffs = offs; length = idx - offs; } metWeight = false; } idx++; } // include end of sequence if (metWeight && (idx - offs) > length) { bestOffs = offs; length = idx - offs; } return cns.substr(bestOffs, length); } void AlnGraphBoost::consensus(std::vector& seqs, int minWeight, size_t minLen) { seqs.clear(); // get the best scoring path std::vector path = bestPath(); // consensus sequence std::string cns; // track the longest consensus path meeting minimum weight int offs = 0, idx = 0; bool metWeight = false; std::vector::iterator curr = path.begin(); for (; curr != path.end(); ++curr) { AlnNode n = *curr; if (n.base == _g[_enterVtx].base || n.base == _g[_exitVtx].base) continue; cns += n.base; // initial beginning of minimum weight section if (!metWeight && n.weight >= minWeight) { offs = idx; metWeight = true; } else if (metWeight && n.weight < minWeight) { // concluded minimum weight section, add sequence to supplied vector metWeight = false; CnsResult result; result.range[0] = offs; result.range[1] = idx; size_t length = idx - offs; result.seq = cns.substr(offs, length); if (length >= minLen) seqs.push_back(result); } idx++; } // include end of sequence if (metWeight) { size_t length = idx - offs; CnsResult result; result.range[0] = offs; result.range[1] = idx; result.seq = cns.substr(offs, length); if (length >= minLen) seqs.push_back(result); } } const std::vector AlnGraphBoost::bestPath() { EdgeIter ei, ee; for (boost::tie(ei, ee) = edges(_g); ei != ee; ++ei) _g[*ei].visited = false; std::map bestNodeScoreEdge; std::map nodeScore; std::queue seedNodes; // start at the end and make our way backwards seedNodes.push(_exitVtx); nodeScore[_exitVtx] = 0.0f; while (true) { if (seedNodes.size() == 0) break; VtxDesc n = seedNodes.front(); seedNodes.pop(); bool bestEdgeFound = false; float bestScore = -FLT_MAX; EdgeDesc bestEdgeD = boost::initialized_value; OutEdgeIter oi, oe; for(boost::tie(oi, oe) = boost::out_edges(n, _g); oi != oe; ++oi) { EdgeDesc outEdgeD = *oi; VtxDesc outNodeD = boost::target(outEdgeD, _g); AlnNode outNode = _g[outNodeD]; float newScore, score = nodeScore[outNodeD]; if (outNode.backbone && outNode.weight == 1) { newScore = score - 10.0f; } else { AlnNode bbNode = _g[_bbMap[outNodeD]]; newScore = _g[outEdgeD].count - bbNode.coverage*0.5f + score; } if (newScore > bestScore) { bestScore = newScore; bestEdgeD = outEdgeD; bestEdgeFound = true; } } if (bestEdgeFound) { nodeScore[n]= bestScore; bestNodeScoreEdge[n] = bestEdgeD; } InEdgeIter ii, ie; for (boost::tie(ii, ie) = boost::in_edges(n, _g); ii != ie; ++ii) { EdgeDesc inEdge = *ii; _g[inEdge].visited = true; VtxDesc inNode = boost::source(inEdge, _g); int notVisited = 0; OutEdgeIter oi, oe; for (boost::tie(oi, oe) = boost::out_edges(inNode, _g); oi != oe; ++oi) { if (_g[*oi].visited == false) notVisited++; } // move onto the target node after we visit all incoming edges for // the target node if (notVisited == 0) seedNodes.push(inNode); } } // construct the final best path VtxDesc prev = _enterVtx, next; std::vector bpath; while (true) { bpath.push_back(_g[prev]); if (bestNodeScoreEdge.count(prev) == 0) { break; } else { EdgeDesc bestOutEdge = bestNodeScoreEdge[prev]; _g[prev].bestOutEdge = bestOutEdge; next = boost::target(bestOutEdge, _g); _g[next].bestInEdge = bestOutEdge; prev = next; } } return bpath; } bool AlnGraphBoost::danglingNodes() { VtxIter curr, last; boost::tie(curr, last) = boost::vertices(_g); bool found = false; for (;curr != last; ++curr) { if (_g[*curr].deleted) continue; if (_g[*curr].base == _g[_enterVtx].base || _g[*curr].base == _g[_exitVtx].base) continue; int indeg = out_degree(*curr, _g); int outdeg = in_degree(*curr, _g); if (outdeg > 0 && indeg > 0) continue; found = true; } return found; } AlnGraphBoost::~AlnGraphBoost(){} canu-1.3/src/utgcns/libpbutgcns/AlnGraphBoost.H000066400000000000000000000176441272335422300215440ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2015-DEC-28 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ // Copyright (c) 2011-2015, Pacific Biosciences of California, Inc. // // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted (subject to the limitations in the // disclaimer below) provided that the following conditions are met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following // disclaimer in the documentation and/or other materials provided // with the distribution. // // * Neither the name of Pacific Biosciences nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE // GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC // BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF // USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT // OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF // SUCH DAMAGE. #ifndef __GCON_ALNGRAPHBOOST_HPP__ #define __GCON_ALNGRAPHBOOST_HPP__ #include /// Alignment graph representation and consensus caller. Based on the original /// Python implementation, pbdagcon. This class is modelled after its /// aligngraph.py component, which accumulates alignment information into a /// partial-order graph and then calls consensus. Used to error-correct pacbio /// on pacbio reads. /// /// Implemented using the boost graph library. // forward declaration //struct Alignment; // this allows me to forward-declare properties with graph descriptors as // members types typedef boost::adjacency_list graphTraits; /// Graph vertex property. An alignment node, which represents one base position /// in the alignment graph. struct AlnNode { char base; ///< DNA base: [ACTG] int coverage; ///< Number of reads align to this position, but not ///< necessarily match int weight; ///< Number of reads that align to this node *with the same base*, but not ///< necessarily represented in the target. bool backbone; ///< Is this node based on the reference bool deleted; ///< mark for removed as part of the merging process graphTraits::edge_descriptor bestInEdge; ///< Best scoring in edge graphTraits::edge_descriptor bestOutEdge; ///< Best scoring out edge AlnNode() { base = 'N'; coverage = 0; weight = 0; backbone = false; deleted = false; } }; /// Graph edge property. Represents an edge between alignment nodes. struct AlnEdge { int count; ///< Number of times this edge was confirmed by an alignment bool visited; ///< Tracks a visit during algorithm processing AlnEdge() { count = 0; visited = false; } }; // Boost-related typedefs // XXX: listS, listS? typedef boost::adjacency_list G; typedef boost::graph_traits::vertex_descriptor VtxDesc; typedef boost::graph_traits::vertex_iterator VtxIter; typedef boost::graph_traits::edge_descriptor EdgeDesc; typedef boost::graph_traits::edge_iterator EdgeIter; typedef boost::graph_traits::in_edge_iterator InEdgeIter; typedef boost::graph_traits::out_edge_iterator OutEdgeIter; typedef boost::property_map::type IndexMap; /// /// Simple consensus interface datastructure /// struct CnsResult { int range[2]; ///< Range on the target std::string seq; ///< Consensus fragment }; /// /// Core alignments into consensus algorithm, implemented using the boost graph /// library. Takes a set of alignments to a reference and builds a higher /// accuracy (~ 99.9) consensus sequence from it. Designed for use in the HGAP /// pipeline as a long read error correction step. /// class AlnGraphBoost { public: /// Constructor. Initialize graph based on the given sequence. Graph is /// annotated with the bases from the backbone. /// \param backbone the reference sequence. AlnGraphBoost(const std::string& backbone); /// Constructor. Initialize graph to a given backbone length. Base /// information is filled in as alignments are added. /// \param blen length of the reference sequence. AlnGraphBoost(const size_t blen); /// Add alignment to the graph. /// \param Alignment an alignment record (see Alignment.hpp) void addAln(dagcon::Alignment& aln); /// Adds a new or increments an existing edge between two aligned bases. /// \param u the 'from' vertex descriptor /// \param v the 'to' vertex descriptor void addEdge(VtxDesc u, VtxDesc v); /// Collapses degenerate nodes (vertices). Must be called before /// consensus(). Calls mergeInNodes() followed by mergeOutNodes(). void mergeNodes(); /// Recursive merge of 'in' nodes. /// \param n the base node to merge around. void mergeInNodes(VtxDesc n); /// Non-recursive merge of 'out' nodes. /// \param n the base node to merge around. void mergeOutNodes(VtxDesc n); /// Mark a given node for removal from graph. Doesn't not modify graph. /// \param n the node to remove. void markForReaper(VtxDesc n); /// Removes the set of nodes that have been marked. Modifies graph. /// Prohibitively expensive when using vecS as the vertex container. void reapNodes(); /// Generates the consensus from the graph. Must be called after /// mergeNodes(). Returns the longest contiguous consensus sequence where /// each base meets the minimum weight requirement. /// \param minWeight sets the minimum weight for each base in the consensus. /// default = 0 const std::string consensus(int minWeight=0); /// Generates all consensus sequences from a target that meet the minimum /// weight requirement. void consensus(std::vector& seqs, int minWeight=0, size_t minLength=500); /// Locates the optimal path through the graph. Called by consensus() const std::vector bestPath(); /// Locate nodes that are missing either in or out edges. bool danglingNodes(); /// Destructor. virtual ~AlnGraphBoost(); private: G _g; VtxDesc _enterVtx; VtxDesc _exitVtx; std::map _bbMap; std::vector _reaperBag; }; #endif // __GCON_ALNGRAPHBOOST_HPP__ canu-1.3/src/utgcns/libpbutgcns/LICENSE000066400000000000000000000036161272335422300177270ustar00rootroot00000000000000#################################################################################$$ # Copyright (c) 2011-2015, Pacific Biosciences of California, Inc. # # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted (subject to the limitations in the # disclaimer below) provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. # # * Neither the name of Pacific Biosciences nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE # GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC # BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED # WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE # DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. #################################################################################$$ canu-1.3/src/utgcns/libpbutgcns/README.md000066400000000000000000000027321272335422300201770ustar00rootroot00000000000000pbutgcns ======== DAG-based consensus caller for celera assemblers unitigs. It is currently tested only using pacbio corrected reads, though in prinicipal it could be used for pacbio un-corrected. It currently uses a simple python adapter to interface with the tigStore. Eventually, this is expected to be replaced with a more direct call to lower-level API calls. Building -------- > make BLASR= > ./pbutgcns -h ### Support libraries needed * [pblibblasr](https://github.com/PacificBiosciences/pblibblasr) BLASR library * [boost](http://www.boost.org/) Popular C++ utility library (1.46 or 1.47) * [log4cpp](http://log4cpp.sourceforge.net/) Logging library (1.0 or 1.1) Running ------- The input is an assembly generated by celera-assembler, typically only through the unitigger stage. However, if you want to compare it to the existing consensus caller, it can be run to completion. Either one will work. A utitilty shell script *pbutgcns_wf.sh* can be used to execute the consensus workflow for a given file containing a list of unitigs ids. Make sure that you have CA's tigStore and gatekeeper utilities, tigStore-adapter.py and pbutgcns in your path. # tmp: temporary directory to generate consensus into # cap: CA-prefix used in celera assembler # nproc: number of threads to use while building consensus # cns: the output file name > tmp=/tmp cap=asm utg=utg.lst nproc=4 cns=utg_consensus.fa pbutgcns_wf.sh canu-1.3/src/utgcns/libpbutgcns/SimpleAligner.C000066400000000000000000000033761272335422300215640ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2015-DEC-28 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include #include #include #include #include #include "Alignment.H" #include "SimpleAligner.H" #include "assert.h" SimpleAligner::SimpleAligner() { } void SimpleAligner::align(dagcon::Alignment &aln, double errorRate) { NDalignment::NDalignResult ndaln; bool aligned = NDalignment::align(aln.qstr.c_str(), aln.qstr.size(), aln.tstr.c_str(), aln.tstr.size(), 150, true, ndaln); if (((double) ndaln._dist / (double) ndaln._size) > errorRate) { aligned = false; } if (aligned) { aln.start += ndaln._tgt_bgn; aln.end = aln.start + ndaln._tgt_end; aln.start++; aln.qstr = std::string(ndaln._qry_aln_str); aln.tstr = std::string(ndaln._tgt_aln_str); } else { aln.start = aln.end = 0; aln.qstr = std::string(); aln.tstr = std::string(); } assert(aln.qstr.length() == aln.tstr.length()); } canu-1.3/src/utgcns/libpbutgcns/SimpleAligner.H000066400000000000000000000021401272335422300215550ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Sergey Koren beginning on 2015-DEC-28 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef __GCON_SIMPLE_ALIGNER__ #define __GCON_SIMPLE_ALIGNER__ #include "dw.H" class SimpleAligner { public: SimpleAligner(); void align(dagcon::Alignment& aln, double errorRate); }; #endif // __GCON_SIMPLE_ALIGNER__ canu-1.3/src/utgcns/stashContains.C000066400000000000000000000162721272335422300173350ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-APR-09 to 2015-MAY-09 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-DEC-01 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "stashContains.H" // Replace the children list in tig with one that has fewer contains. The original // list is returned. savedChildren * stashContains(tgTig *tig, double maxCov, bool beVerbose) { if (tig->numberOfChildren() == 1) return(NULL); // Stats we report int32 nOrig = tig->numberOfChildren(); int32 nBack = 0; int32 nCont = 0; int32 nSave = 0; int64 nBase = 0; int64 nBaseDove = 0; int64 nBaseCont = 0; int64 nBaseSave = 0; // Save the original children savedChildren *saved = new savedChildren(tig); bool *isBack = new bool [nOrig]; // True, we save the child for processing readLength *posLen = new readLength [nOrig]; // Sorting by length of child // Sort the original children by position. std::sort(saved->children, saved->children + saved->childrenLen); // The first read is always saved int32 loEnd = saved->children[0].min(); int32 hiEnd = saved->children[0].max(); isBack[0] = 1; nBack = 1; posLen[0].idx = 0; posLen[0].len = hiEnd - loEnd; nBaseDove += posLen[0].len; nBase += posLen[0].len; // For the other reads, save it if it extends the backbone sequence. for (uint32 fi=1; fichildren[fi].min(); int32 hi = saved->children[fi].max(); posLen[fi].idx = fi; posLen[fi].len = hi - lo; nBase += posLen[fi].len; if (hi <= hiEnd) { isBack[fi] = false; nCont++; nBaseCont += posLen[fi].len; } else { isBack[fi] = true; nBack++; nBaseDove += posLen[fi].len; } hiEnd = MAX(hi, hiEnd); } // Entertain the user with some statistics double totlCov = (double)nBase / hiEnd; saved->numContains = nCont; saved->covContain = (double)nBaseCont / hiEnd; saved->percContain = 100.0 * nBaseCont / nBase;; saved->numDovetails = nBack; saved->covDovetail = (double)nBaseDove / hiEnd; saved->percDovetail = 100.0 * nBaseDove / nBase;; if (beVerbose) saved->reportDetected(stderr, tig->tigID()); // If the tig has more coverage than allowed, throw out some of the contained reads. if ((totlCov >= maxCov) && (maxCov > 0)) { std::sort(posLen, posLen + nOrig, greater()); // Sort by length, larger first nBaseSave = 0.0; for (uint32 ii=0; ii < nOrig; ii++) { if (ii > 0) assert(posLen[ii-1].len >= posLen[ii].len); if (isBack[posLen[ii].idx]) // Already a backbone read. continue; if ((double)(nBaseSave + nBaseDove) / hiEnd < maxCov) { isBack[posLen[ii].idx] = true; // Save it. nSave++; nBaseSave += posLen[ii].len; } } saved->numContainsRemoved = nOrig - nBack - nSave; saved->covContainsRemoved = (double)(nBaseCont - nBaseSave) / hiEnd; saved->numContainsSaved = nSave; saved->covContainsSaved = (double)nBaseSave / hiEnd; if (beVerbose) saved->reportRemoved(stderr, tig->tigID()); // For all the reads we saved, copy them to a new children list in the tig tig->_childrenLen = 0; tig->_childrenMax = nBack + nSave; tig->_children = new tgPosition [tig->_childrenMax]; // The original is in savedChildren now for (uint32 fi=0; fichildren[fi].ident(), saved->children[fi].bgn(), children[fi].end()); tig->_children[tig->_childrenLen++] = saved->children[fi]; } } // Else, the tig coverage is acceptable and we do no filtering. else { delete saved; saved = NULL; } delete [] isBack; delete [] posLen; return(saved); } // Restores the f_list, and updates the position of non-contained reads. // void unstashContains(tgTig *tig, savedChildren *saved) { if (saved == NULL) return; uint32 oldMax = 0; uint32 newMax = 0; // For fragments not involved in the consensus computation, we'll scale their position linearly // from the old max to the new max. // // We probably should do an alignment to the consensus sequence to find the true location, but // that's (a) expensive and (b) likely overkill for these unitigs. // Find the oldMax for (uint32 fi=0, ci=0; fichildrenLen; fi++) if (oldMax < saved->children[fi].max()) oldMax = saved->children[fi].max(); // Find the newMax // We could have just done: newMax = tig->gappedLength(); for (uint32 fi=0, ci=0; finumberOfChildren(); fi++) if (newMax < tig->getChild(fi)->max()) newMax = tig->getChild(fi)->max(); double sf = (double)newMax / oldMax; // First, we need a map from the child id to the location in the current tig map idmap; for (uint32 ci=0; ci < tig->numberOfChildren(); ci++) idmap[tig->getChild(ci)->ident()] = tig->getChild(ci); // Now, over all the reads in the original saved fragment list, update the position. Either from // the computed result, or by extrapolating. for (uint32 fi=0; fichildrenLen; fi++) { uint32 iid = saved->children[fi].ident(); // Does the ID exist in the new positions? Copy the new position to the original list. if (idmap.find(iid) != idmap.end()) { saved->children[fi] = *idmap[iid]; idmap.erase(iid); } // Otherwise, fudge the positions. else { int32 nmin = sf * saved->children[fi].min(); int32 nmax = sf * saved->children[fi].max(); if (nmin > newMax) nmin = newMax; if (nmax > newMax) nmax = newMax; saved->children[fi].setMinMax(nmin, nmax); } } if (idmap.empty() == false) fprintf(stderr, "Failed to unstash the contained reads. Still have "F_SIZE_T" reads unplaced.\n", idmap.size()); assert(idmap.empty() == true); // Throw out the reduced list, and restore the original. delete [] tig->_children; tig->_childrenLen = saved->childrenLen; tig->_childrenMax = saved->childrenMax; tig->_children = saved->children; } canu-1.3/src/utgcns/stashContains.H000066400000000000000000000062671272335422300173450ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * Modifications by: * * Brian P. Walenz from 2015-APR-09 to 2015-MAY-09 * are Copyright 2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2016-JAN-06 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #ifndef STASH_CONTAINS_H #define STASH_CONTAINS_H #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" #include #include class readLength { public: uint32 idx; int32 len; bool operator<(const readLength &that) const { return(len < that.len); }; bool operator>(const readLength &that) const { return(len > that.len); }; }; class savedChildren { public: savedChildren(tgTig *tig) { childrenLen = tig->_childrenLen; childrenMax = tig->_childrenMax; children = tig->_children; numContains = 0; covContain = 0.0; percContain = 0.0; numContainsRemoved = 0; covContainsRemoved = 0.0; numContainsSaved = 0; covContainsSaved = 0.0; numDovetails = 0; covDovetail = 0.0; percDovetail = 0.0; }; void reportDetected(FILE *out, uint32 id) { fprintf(out, " unitig %d detected "F_S32" contains (%.2fx, %.2f%%) "F_S32" dovetail (%.2fx, %.2f%%)\n", id, numContains, covContain, percContain, numDovetails, covDovetail, percDovetail); }; void reportRemoved(FILE *out, uint32 id) { fprintf(out, " unitig %d removing "F_S32" (%.2fx) contained reads; processing only "F_S32" contained (%.2fx) and "F_S32" dovetail (%.2fx) reads\n", id, numContainsRemoved, covContainsRemoved, numContainsSaved, covContainsSaved, numDovetails, covDovetail); }; // The saved children proper. uint32 childrenLen; uint32 childrenMax; tgPosition *children; // Stats on the filtering, for logging by the caller uint32 numContains; double covContain; double percContain; uint32 numContainsRemoved; double covContainsRemoved; uint32 numContainsSaved; double covContainsSaved; uint32 numDovetails; double covDovetail; double percDovetail; }; savedChildren * stashContains(tgTig *tig, double maxCov, bool beVerbose = false); void unstashContains(tgTig *tig, savedChildren *saved); #endif // STASH_CONTAINS_H canu-1.3/src/utgcns/utgcns.C000066400000000000000000000512051272335422300160120ustar00rootroot00000000000000 /****************************************************************************** * * This file is part of canu, a software program that assembles whole-genome * sequencing reads into contigs. * * This software is based on: * 'Celera Assembler' (http://wgs-assembler.sourceforge.net) * the 'kmer package' (http://kmer.sourceforge.net) * both originally distributed by Applera Corporation under the GNU General * Public License, version 2. * * Canu branched from Celera Assembler at its revision 4587. * Canu branched from the kmer project at its revision 1994. * * This file is derived from: * * src/AS_CNS/utgcns.C * * Modifications by: * * Brian P. Walenz from 2009-OCT-05 to 2014-MAR-31 * are Copyright 2009-2014 J. Craig Venter Institute, and * are subject to the GNU General Public License version 2 * * Brian P. Walenz from 2014-DEC-30 to 2015-AUG-07 * are Copyright 2014-2015 Battelle National Biodefense Institute, and * are subject to the BSD 3-Clause License * * Brian P. Walenz beginning on 2015-OCT-09 * are a 'United States Government Work', and * are released in the public domain * * Sergey Koren beginning on 2015-DEC-28 * are a 'United States Government Work', and * are released in the public domain * * File 'README.licenses' in the root directory of this distribution contains * full conditions and disclaimers for each license. */ #include "AS_global.H" #include "gkStore.H" #include "tgStore.H" #include "AS_UTL_decodeRange.H" #include "stashContains.H" #include "unitigConsensus.H" #include #include #include int main (int argc, char **argv) { char *gkpName = NULL; char *tigName = NULL; uint32 tigVers = UINT32_MAX; uint32 tigPart = UINT32_MAX; char *tigFileName = NULL; uint32 utgBgn = UINT32_MAX; uint32 utgEnd = UINT32_MAX; char *outResultsName = NULL; char *outLayoutsName = NULL; char *outSeqNameA = NULL; char *outSeqNameQ = NULL; char *outPackageName = NULL; FILE *outResultsFile = NULL; FILE *outLayoutsFile = NULL; FILE *outSeqFileA = NULL; FILE *outSeqFileQ = NULL; FILE *outPackageFile = NULL; char *inPackageName = NULL; char algorithm = 'P'; uint32 numThreads = 0; bool forceCompute = false; double errorRate = 0.12; double errorRateMax = 0.40; uint32 minOverlap = 40; int32 numFailures = 0; bool showResult = false; double maxCov = 0.0; uint32 maxLen = UINT32_MAX; uint32 verbosity = 0; argc = AS_configure(argc, argv); int arg=1; int err=0; while (arg < argc) { if (strcmp(argv[arg], "-G") == 0) { gkpName = argv[++arg]; } else if (strcmp(argv[arg], "-T") == 0) { tigName = argv[++arg]; tigVers = atoi(argv[++arg]); tigPart = atoi(argv[++arg]); if (argv[arg][0] == '.') tigPart = UINT32_MAX; if (tigVers == 0) fprintf(stderr, "invalid tigStore version (-T store version partition) '-t %s %s %s'.\n", argv[arg-2], argv[arg-1], argv[arg]), exit(1); if (tigPart == 0) fprintf(stderr, "invalid tigStore partition (-T store version partition) '-t %s %s %s'.\n", argv[arg-2], argv[arg-1], argv[arg]), exit(1); } else if (strcmp(argv[arg], "-u") == 0) { AS_UTL_decodeRange(argv[++arg], utgBgn, utgEnd); } else if (strcmp(argv[arg], "-t") == 0) { tigFileName = argv[++arg]; } else if (strcmp(argv[arg], "-O") == 0) { outResultsName = argv[++arg]; } else if (strcmp(argv[arg], "-L") == 0) { outLayoutsName = argv[++arg]; } else if (strcmp(argv[arg], "-A") == 0) { outSeqNameA = argv[++arg]; } else if (strcmp(argv[arg], "-Q") == 0) { outSeqNameQ = argv[++arg]; } else if (strcmp(argv[arg], "-quick") == 0) { algorithm = 'Q'; } else if (strcmp(argv[arg], "-pbdagcon") == 0) { algorithm = 'P'; } else if (strcmp(argv[arg], "-utgcns") == 0) { algorithm = 'U'; } else if (strcmp(argv[arg], "-threads") == 0) { numThreads = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-p") == 0) { inPackageName = argv[++arg]; } else if (strcmp(argv[arg], "-P") == 0) { outPackageName = argv[++arg]; } else if (strcmp(argv[arg], "-e") == 0) { errorRate = atof(argv[++arg]); } else if (strcmp(argv[arg], "-em") == 0) { errorRateMax = atof(argv[++arg]); } else if (strcmp(argv[arg], "-l") == 0) { minOverlap = atoi(argv[++arg]); } else if (strcmp(argv[arg], "-f") == 0) { forceCompute = true; } else if (strcmp(argv[arg], "-v") == 0) { showResult = true; } else if (strcmp(argv[arg], "-V") == 0) { verbosity++; } else if (strcmp(argv[arg], "-maxcoverage") == 0) { maxCov = atof(argv[++arg]); } else if (strcmp(argv[arg], "-maxlength") == 0) { maxLen = atof(argv[++arg]); } else { fprintf(stderr, "%s: Unknown option '%s'\n", argv[0], argv[arg]); err++; } arg++; } if ((gkpName == NULL) && (inPackageName == NULL)) err++; if ((tigFileName == NULL) && (tigName == NULL) && (inPackageName == NULL)) err++; if (err) { fprintf(stderr, "usage: %s [opts]\n", argv[0]); fprintf(stderr, "\n"); fprintf(stderr, " INPUT\n"); fprintf(stderr, " -G g Load reads from gkStore 'g'\n"); fprintf(stderr, " -T t v p Load unitigs from tgStore 't', version 'v', partition 'p'.\n"); fprintf(stderr, " Expects reads will be in gkStore partition 'p' as well\n"); fprintf(stderr, " Use p='.' to specify no partition\n"); fprintf(stderr, " -t file Test the computation of the unitig layout in 'file'\n"); fprintf(stderr, " 'file' can be from:\n"); fprintf(stderr, " 'tgStoreDump -d layout' (human readable layout format)\n"); fprintf(stderr, " 'utgcns -L' (human readable layout format)\n"); fprintf(stderr, " 'utgcns -O' (binary multialignment format)\n"); fprintf(stderr, "\n"); fprintf(stderr, " -p package Load unitig and read from 'package' created with -P. This\n"); fprintf(stderr, " is usually used by developers.\n"); fprintf(stderr, "\n"); fprintf(stderr, "\n"); fprintf(stderr, " ALGORITHM\n"); fprintf(stderr, " -quick No alignments, just paste read sequence into the unitig positions.\n"); fprintf(stderr, " This is very fast, but the consensus sequence is formed from a mosaic\n"); fprintf(stderr, " of read sequences, and there can be large indel. This is useful for\n"); fprintf(stderr, " checking intermediate assembly structure by mapping to reference, or\n"); fprintf(stderr, " possibly for use as input to a polishing step.\n"); fprintf(stderr, " -pbdagcon Use pbdagcon (https://github.com/PacificBiosciences/pbdagcon).\n"); fprintf(stderr, " This is fast and robust. It is the default algorithm. It does not\n"); fprintf(stderr, " generate a final multialignment output (the -v option will not show\n"); fprintf(stderr, " anything useful).\n"); fprintf(stderr, " -utgcns Use utgcns (the original Celera Assembler consensus algorithm)\n"); fprintf(stderr, " This isn't as fast, isn't as robust, but does generate a final multialign\n"); fprintf(stderr, " output.\n"); fprintf(stderr, "\n"); fprintf(stderr, "\n"); fprintf(stderr, " OUTPUT\n"); fprintf(stderr, " -O results Write computed tigs to binary output file 'results'\n"); fprintf(stderr, " -L layouts Write computed tigs to layout output file 'layouts'\n"); fprintf(stderr, " -A fasta Write computed tigs to fasta output file 'fasta'\n"); fprintf(stderr, " -Q fastq Write computed tigs to fastq output file 'fastq'\n"); fprintf(stderr, "\n"); fprintf(stderr, " -P package Create a copy of the inputs needed to compute the unitigs. This\n"); fprintf(stderr, " file can then be sent to the developers for debugging. The unitig(s)\n"); fprintf(stderr, " are not processed and no other outputs are created. Ideally,\n"); fprintf(stderr, " only one unitig is selected (-u, below).\n"); fprintf(stderr, "\n"); fprintf(stderr, " TIG SELECTION (if -T input is used)\n"); fprintf(stderr, " -u b Compute only unitig ID 'b' (must be in the correct partition!)\n"); fprintf(stderr, " -u b-e Compute only unitigs from ID 'b' to ID 'e'\n"); fprintf(stderr, " -f Recompute unitigs that already have a multialignment\n"); fprintf(stderr, " -maxlength l Do not compute consensus for unitigs longer than l bases.\n"); fprintf(stderr, "\n"); fprintf(stderr, " PARAMETERS\n"); fprintf(stderr, " -e e Expect alignments at up to fraction e error\n"); fprintf(stderr, " -em m Don't ever allow alignments more than fraction m error\n"); fprintf(stderr, " -l l Expect alignments of at least l bases\n"); fprintf(stderr, " -maxcoverage c Use non-contained reads and the longest contained reads, up to\n"); fprintf(stderr, " C coverage, for consensus generation. The default is 0, and will\n"); fprintf(stderr, " use all reads.\n"); fprintf(stderr, "\n"); fprintf(stderr, " LOGGING\n"); fprintf(stderr, " -v Show multialigns.\n"); fprintf(stderr, " -V Enable debugging option 'verbosemultialign'.\n"); fprintf(stderr, "\n"); if ((gkpName == NULL) && (inPackageName == NULL)) fprintf(stderr, "ERROR: No gkpStore (-G) and no package (-p) supplied.\n"); if ((tigFileName == NULL) && (tigName == NULL) && (inPackageName == NULL)) fprintf(stderr, "ERROR: No tigStore (-T) OR no test unitig (-t) OR no package (-p) supplied.\n"); exit(1); } errno = 0; // Open output files. If we're creating a package, the usual output files are not opened. if (outPackageName) outPackageFile = fopen(outPackageName, "w"); if (errno) fprintf(stderr, "Failed to open output package file '%s': %s\n", outPackageName, strerror(errno)), exit(1); if ((outResultsName) && (outPackageName == NULL)) outResultsFile = fopen(outResultsName, "w"); if (errno) fprintf(stderr, "Failed to open output results file '%s': %s\n", outResultsName, strerror(errno)), exit(1); if ((outLayoutsName) && (outPackageName == NULL)) outLayoutsFile = fopen(outLayoutsName, "w"); if (errno) fprintf(stderr, "Failed to open output layout file '%s': %s\n", outLayoutsName, strerror(errno)), exit(1); if ((outSeqNameA) && (outPackageName == NULL)) outSeqFileA = fopen(outSeqNameA, "w"); if (errno) fprintf(stderr, "Failed to open output FASTA file '%s': %s\n", outSeqNameA, strerror(errno)), exit(1); if ((outSeqNameQ) && (outPackageName == NULL)) outSeqFileQ = fopen(outSeqNameQ, "w"); if (errno) fprintf(stderr, "Failed to open output FASTQ file '%s': %s\n", outSeqNameQ, strerror(errno)), exit(1); if (numThreads > 0) { omp_set_num_threads(numThreads); fprintf(stderr, "number of threads = %d (command line)\n", numThreads); fprintf(stderr, "\n"); } else { fprintf(stderr, "number of threads = %d (OpenMP default)\n", omp_get_max_threads()); fprintf(stderr, "\n"); } // Open gatekeeper for read only, and load the partitioned data if tigPart > 0. gkStore *gkpStore = NULL; tgStore *tigStore = NULL; FILE *tigFile = NULL; FILE *inPackageFile = NULL; map *inPackageRead = NULL; map *inPackageReadData = NULL; if (gkpName) { fprintf(stderr, "-- Opening gkpStore '%s' partition %u.\n", gkpName, tigPart); gkpStore = gkStore::gkStore_open(gkpName, gkStore_readOnly, tigPart); } if (tigName) { fprintf(stderr, "-- Opening tigStore '%s' version %u.\n", tigName, tigVers); tigStore = new tgStore(tigName, tigVers); } if (tigFileName) { fprintf(stderr, "-- Opening tigFile '%s'.\n", tigFileName); errno = 0; tigFile = fopen(tigFileName, "r"); if (errno) fprintf(stderr, "Failed to open input tig file '%s': %s\n", tigFileName, strerror(errno)), exit(1); } if (inPackageName) { fprintf(stderr, "-- Opening package file '%s'.\n", inPackageName); errno = 0; inPackageFile = fopen(inPackageName, "r"); if (errno) fprintf(stderr, "Failed to open input package file '%s': %s\n", inPackageName, strerror(errno)), exit(1); } // Report some sizes. fprintf(stderr, "sizeof(abBead) "F_SIZE_T"\n", sizeof(abBead)); fprintf(stderr, "sizeof(abColumn) "F_SIZE_T"\n", sizeof(abColumn)); fprintf(stderr, "sizeof(abAbacus) "F_SIZE_T"\n", sizeof(abAbacus)); fprintf(stderr, "sizeof(abSequence) "F_SIZE_T"\n", sizeof(abSequence)); // Decide on what to compute. Either all unitigs, or a single unitig, or a special case test. uint32 b = 0; uint32 e = UINT32_MAX; if (tigStore) { if (utgEnd > tigStore->numTigs() - 1) utgEnd = tigStore->numTigs() - 1; if (utgBgn != UINT32_MAX) { b = utgBgn; e = utgEnd; } else { b = 0; e = utgEnd; } fprintf(stderr, "-- Computing unitig consensus for b="F_U32" to e="F_U32" with errorRate %0.4f (max %0.4f) and minimum overlap "F_U32"\n", b, e, errorRate, errorRateMax, minOverlap); } else { fprintf(stderr, "-- Computing unitig consensus with errorRate %0.4f (max %0.4f) and minimum overlap "F_U32"\n", errorRate, errorRateMax, minOverlap); } fprintf(stderr, "\n"); // I don't like this loop control. for (uint32 ti=b; (e == UINT32_MAX) || (ti <= e); ti++) { tgTig *tig = NULL; // If a tigStore, load the tig. The tig is the owner; it cannot be deleted by us. if (tigStore) tig = tigStore->loadTig(ti); // If a tigFile or a package, create a new tig and fill it. Obviously, we own it. if (tigFile || inPackageFile) { tig = new tgTig(); if (tig->loadFromStreamOrLayout((tigFile != NULL) ? tigFile : inPackageFile) == false) { delete tig; break; } } // No tig loaded, keep going. if (tig == NULL) continue; // If a package, populate the read and readData maps with data from the package. if (inPackageFile) { inPackageRead = new map; inPackageReadData = new map; for (int32 ii=0; iinumberOfChildren(); ii++) { uint32 readID = tig->getChild(ii)->ident(); gkRead *read = (*inPackageRead)[readID] = new gkRead; gkReadData *data = (*inPackageReadData)[readID] = new gkReadData; gkStore::gkStore_loadReadFromStream(inPackageFile, read, data); if (read->gkRead_readID() != readID) fprintf(stderr, "ERROR: package not in sync with tig. package readID = %u tig readID = %u\n", read->gkRead_readID(), readID); assert(read->gkRead_readID() == readID); } } // More 'not liking' - set the verbosity level for logging. tig->_utgcns_verboseLevel = verbosity; // Are we parittioned? Is this tig in our partition? if (tigPart != UINT32_MAX) { uint32 missingReads = 0; for (uint32 ii=0; iinumberOfChildren(); ii++) if (gkpStore->gkStore_getReadInPartition(tig->getChild(ii)->ident()) == NULL) missingReads++; if (missingReads) { //fprintf(stderr, "SKIP unitig %u with %u reads found only %u reads in partition, skipped\n", // tig->tigID(), tig->numberOfChildren(), tig->numberOfChildren() - missingReads); continue; } } if (tig->length(true) > maxLen) { fprintf(stderr, "SKIP unitig %d of length %d (%d children) - too long, skipped\n", tig->tigID(), tig->length(true), tig->numberOfChildren()); continue; } if (tig->numberOfChildren() == 0) { fprintf(stderr, "SKIP unitig %d of length %d (%d children) - no children, skipped\n", tig->tigID(), tig->length(true), tig->numberOfChildren()); continue; } bool exists = tig->consensusExists(); if (tig->numberOfChildren() > 1) fprintf(stderr, "Working on unitig %d of length %d (%d children)%s%s\n", tig->tigID(), tig->length(true), tig->numberOfChildren(), ((exists == true) && (forceCompute == false)) ? " - already computed" : "", ((exists == true) && (forceCompute == true)) ? " - already computed, recomputing" : ""); // Process the tig. Remove deep coverage, create a consensus object, process it, and report the results. // before we add it to the store. unitigConsensus *utgcns = new unitigConsensus(gkpStore, errorRate, errorRateMax, minOverlap); savedChildren *origChildren = NULL; bool success = exists; // Save the tig in the package? // // The original idea was to dump the tig and all the reads, then load the tig and process as normal. // Sadly, stashContains() rearranges the order of the reads even if it doesn't remove any. The rearranged // tig couldn't be saved (otherwise it would be rearranged again). So, we were in the position of // needing to save the original tig and the rearranged reads. Impossible. // // Instead, we save the origianl tig and original reads -- including any that get stashed -- then // load them all back into a map for use in consensus proper. It's a bit of a pain, and could // have way more reads saved than necessary. if (outPackageFile) { utgcns->savePackage(outPackageFile, tig); fprintf(stderr, " Packaged unitig %u into '%s'\n", tig->tigID(), outPackageName); } // Compute consensus if it doesn't exist, or if we're forcing a recompute. But only if we // didn't just package it. if ((outPackageFile == NULL) && ((exists == false) || (forceCompute == true))) { origChildren = stashContains(tig, maxCov, true); switch (algorithm) { case 'Q': success = utgcns->generateQuick(tig, inPackageRead, inPackageReadData); break; case 'P': default: success = utgcns->generatePBDAG(tig, inPackageRead, inPackageReadData); break; case 'U': success = utgcns->generate(tig, inPackageRead, inPackageReadData); break; } } // If it was successful (or existed already), output. Success is always false if the unitig // was packaged, regardless of if it existed already. if (success == true) { if ((showResult) && (gkpStore)) // No gkpStore if we're from a package. Dang. tig->display(stdout, gkpStore, 200, 3); unstashContains(tig, origChildren); if (outResultsFile) tig->saveToStream(outResultsFile); if (outLayoutsFile) tig->dumpLayout(outLayoutsFile); if (outSeqFileA) tig->dumpFASTA(outSeqFileA, true); if (outSeqFileQ) tig->dumpFASTQ(outSeqFileQ, true); } // Report failures. if ((success == false) && (outPackageFile == NULL)) { fprintf(stderr, "unitigConsensus()-- unitig %d failed.\n", tig->tigID()); numFailures++; } // Clean up, unloading or deleting the tig. delete utgcns; // No real reason to keep this until here. delete origChildren; // Need to keep it until after we display() above. if (tigStore) tigStore->unloadTig(tig->tigID(), true); // Tell the store we're done with it if (tigFile) delete tig; } finish: delete tigStore; gkpStore->gkStore_close(); if (tigFile) fclose(tigFile); if (outResultsFile) fclose(outResultsFile); if (outLayoutsFile) fclose(outLayoutsFile); if (outPackageFile) fclose(outPackageFile); if (inPackageFile) fclose(inPackageFile); if (numFailures) { fprintf(stderr, "WARNING: Total number of unitig failures = %d\n", numFailures); fprintf(stderr, "\n"); fprintf(stderr, "Consensus did NOT finish successfully.\n"); } else { fprintf(stderr, "Consensus finished successfully. Bye.\n"); } return(numFailures != 0); } canu-1.3/src/utgcns/utgcns.mk000066400000000000000000000010171272335422300162330ustar00rootroot00000000000000# If 'make' isn't run from the root directory, we need to set these to # point to the upper level build directory. ifeq "$(strip ${BUILD_DIR})" "" BUILD_DIR := ../$(OSTYPE)-$(MACHINETYPE)/obj endif ifeq "$(strip ${TARGET_DIR})" "" TARGET_DIR := ../$(OSTYPE)-$(MACHINETYPE)/bin endif TARGET := utgcns SOURCES := utgcns.C stashContains.C SRC_INCDIRS := .. ../AS_UTL ../stores libcns libpbutgcns libNDFalcon libboost TGT_LDFLAGS := -L${TARGET_DIR} TGT_LDLIBS := -lcanu TGT_PREREQS := libcanu.a SUBMAKEFILES :=