edirect/0000755000725300123250000000000013041425773012512 5ustar coremakecppcoreedirect/efetch0000755000725300123250000000040712122445476013700 0ustar coremakecppcore#!/bin/sh dir=`dirname "$0"` case "`uname -s`" in CYGWIN_NT*) # Use a negative match here because the shell treats 0 as success. if perl -e 'exit $^O !~ /^MSWin/'; then dir=`cygpath -w "$dir"` fi ;; esac exec perl "$dir"/edirect.pl -fetch "$@" edirect/setup.sh0000755000725300123250000000363613002741371014211 0ustar coremakecppcore#!/bin/bash -norc DIR="$( cd "$( dirname "$0" )" && pwd )" cat <> "$DIR/_cpan/CPAN/MyConfig.pm" perl -I"$DIR/_cpan" "$DIR/setup-deps.pl" "$DIR/setup-deps.log" 2>&1 rm -rf "$DIR/_cpan" cd "$DIR" if ! perl -Iaux/lib/perl5 -MMozilla::CA -e '1;' 2>/dev/null then gzip -cd Mozilla-CA.tar.gz | tar xvf - fi osname=`uname -s | sed -e 's/[0-9.-]*$//'` cputype=`uname -m` case "$osname-$cputype" in Linux-x86_64 | Darwin-x86_64 | CYGWIN_NT-x86_64 ) ./ftp-cp ftp.ncbi.nlm.nih.gov /entrez/entrezdirect xtract."$osname".gz gunzip -f xtract."$osname".gz ;; esac if [ -f xtract."$osname" ] then chmod +x xtract."$osname" else echo "Unable to download a prebuilt xtract executable; attempting to" echo "build one from xtract.go. A Perl fallback is also available, and" echo "will be used if necessary, so please disregard any errors below." go build -o xtract."$osname" xtract.go fi echo "" echo "Entrez Direct has been successfully downloaded and installed." echo "" prfx="In order to complete the configuration process, please execute the following:\n" target=bash_profile if ! grep "$target" "$HOME/.bashrc" >/dev/null 2>&1 then if [ ! -f $HOME/.$target ] || grep 'bashrc' "$HOME/.$target" >/dev/null 2>&1 then target=bashrc else if [ -n "$prfx" ] then echo -e "$prfx" prfx="" fi echo " echo \"source ~/.bash_profile\" >>" "\$HOME/.bashrc" fi fi if ! grep "PATH.*edirect" "$HOME/.$target" >/dev/null 2>&1 then if [ -n "$prfx" ] then echo -e "$prfx" prfx="" fi echo " echo \"export PATH=\\\$PATH:$DIR\" >>" "\$HOME/.$target" fi if [ -z "$prfx" ] then echo "" echo "or manually edit the PATH variable assignment in your .bash_profile file." echo "" fi edirect/efilter0000755000725300123250000000041012122445476014066 0ustar coremakecppcore#!/bin/sh dir=`dirname "$0"` case "`uname -s`" in CYGWIN_NT*) # Use a negative match here because the shell treats 0 as success. if perl -e 'exit $^O !~ /^MSWin/'; then dir=`cygpath -w "$dir"` fi ;; esac exec perl "$dir"/edirect.pl -filter "$@" edirect/eaddress0000755000725300123250000000041112453604660014226 0ustar coremakecppcore#!/bin/sh dir=`dirname "$0"` case "`uname -s`" in CYGWIN_NT*) # Use a negative match here because the shell treats 0 as success. if perl -e 'exit $^O !~ /^MSWin/'; then dir=`cygpath -w "$dir"` fi ;; esac exec perl "$dir"/edirect.pl -address "$@" edirect/espell0000755000725300123250000000040712403410275013715 0ustar coremakecppcore#!/bin/sh dir=`dirname "$0"` case "`uname -s`" in CYGWIN_NT*) # Use a negative match here because the shell treats 0 as success. if perl -e 'exit $^O !~ /^MSWin/'; then dir=`cygpath -w "$dir"` fi ;; esac exec perl "$dir"/edirect.pl -spell "$@" edirect/setup-deps.pl0000755000725300123250000000322612750437512015146 0ustar coremakecppcore#!/usr/bin/perl -w use strict; use CPAN::MyConfig; use CPAN; use CPAN::HandleConfig; use CPAN::Shell; my $root; BEGIN { $root = $INC[0]; if ($root !~ s:/_cpan$::) { die "Please invoke this script via .../setup.sh." } if ( ! exists $CPAN::Config{cpan_home} ) { $CPAN::Config = { 'auto_commit' => 1, 'cpan_home' => "$root/_cpan", 'ftp_passive' => 1, 'install_help' => 'manual', 'urllist' => [@CPAN::Defaultsites] }; } # $CPAN::DEBUG ||= $CPAN::DEBUG{'FTP'}; CPAN::HandleConfig->load(autoconfig => 1, auto_pick => 1, doit => 1); CPAN::Shell::setup_output; CPAN::Index->reload; my $ll = CPAN::Shell->expandany('local::lib'); if ( ( ! $ll->inst_file || $ll->inst_version =~ /^1\./) && ! -d "$root/aux/lib/perl5/local" ) { $ll->get; system('mkdir', '-p', "$root/aux/lib/perl5/local"); system('cp', $ll->distribution->dir . "/lib/local/lib.pm", "$root/aux/lib/perl5/local/lib.pm"); } } use lib "$root/aux/lib/perl5"; use local::lib("$root/aux", '--no-create'); my @lwp_deps = qw(Encode::Locale File::Listing HTML::Parser HTML::Tagset HTML::Tree HTTP:Cookies HTTP::Date HTTP::Message HTTP::Negotiate LWP::MediaTypes LWP::Protocol::https Net::HTTP URI WWW::RobotRules Mozilla::CA); for my $module (@lwp_deps, 'Time::HiRes') { if ( ! CPAN::Shell->expandany($module)->inst_file ) { CPAN::Shell->install($module); } } if ( ! CPAN::Shell->expandany('LWP')->inst_file ) { CPAN::Shell->install('Bundle::LWP'); } edirect/eproxy0000755000725300123250000000040712122445476013770 0ustar coremakecppcore#!/bin/sh dir=`dirname "$0"` case "`uname -s`" in CYGWIN_NT*) # Use a negative match here because the shell treats 0 as success. if perl -e 'exit $^O !~ /^MSWin/'; then dir=`cygpath -w "$dir"` fi ;; esac exec perl "$dir"/edirect.pl -proxy "$@" edirect/filter-stop-words0000755000725300123250000000160713014740172016041 0ustar coremakecppcore#!/bin/bash -norc stop_words="#a#about#again#all#almost#also#although#always#among#an#and#\ another#any#are#as#at#be#because#been#before#being#between#both#but#by#can#\ could#did#do#does#done#due#during#each#either#enough#especially#etc#for#\ found#from#further#had#has#have#having#here#how#however#i#if#in#into#is#it#\ its#itself#just#kg#km#made#mainly#make#may#mg#might#ml#mm#most#mostly#\ must#nearly#neither#no#nor#obtained#of#often#on#our#overall#perhaps#pmid#\ quite#rather#really#regarding#seem#seen#several#should#show#showed#shown#\ shows#significantly#since#so#some#such#than#that#the#their#theirs#them#\ then#there#therefore#these#they#this#those#through#thus#to#upon#use#used#\ using#various#very#was#we#were#what#when#which#while#with#within#without#would#" while read line do if [[ $stop_words != *"#$line#"* ]] then echo "$line" elif [ -n "$*" ] then echo "$*" fi done edirect/xtract.go0000644000725300123250000061511213041323651014344 0ustar coremakecppcore// =========================================================================== // // PUBLIC DOMAIN NOTICE // National Center for Biotechnology Information (NCBI) // // This software/database is a "United States Government Work" under the // terms of the United States Copyright Act. It was written as part of // the author's official duties as a United States Government employee and // thus cannot be copyrighted. This software/database is freely available // to the public for use. The National Library of Medicine and the U.S. // Government do not place any restriction on its use or reproduction. // We would, however, appreciate having the NCBI and the author cited in // any work or product based on this material. // // Although all reasonable efforts have been taken to ensure the accuracy // and reliability of the software and data, the NLM and the U.S. // Government do not and cannot warrant the performance or results that // may be obtained by using this software or data. The NLM and the U.S. // Government disclaim all warranties, express or implied, including // warranties of performance, merchantability or fitness for any particular // purpose. // // =========================================================================== // // File Name: xtract.go // // Author: Jonathan Kans // // ========================================================================== /* test for presence of go compiler, cross-compile xtract executables, and pack into archive, by running: if hash go 2>/dev/null then env GOOS=darwin GOARCH=amd64 go build -o xtract.Darwin -v xtract.go env GOOS=linux GOARCH=amd64 go build -o xtract.Linux -v xtract.go env GOOS=windows GOARCH=386 go build -o xtract.CYGWIN_NT -v xtract.go tar -czf archive.tar.gz xtract.[A-Z]* rm xtract.[A-Z]* fi */ package main import ( "bytes" "container/heap" "fmt" "html" "io" "math" "os" "path" "runtime" "runtime/debug" "runtime/pprof" "strconv" "strings" "sync" "time" "unicode" ) // VERSION AND HELP MESSAGE TEXT const xtractVersion = "6.10" const xtractHelp = ` Overview Xtract uses command-line arguments to convert XML data into a tab-delimited table. -pattern places the data from individual records into separate rows. -element extracts values from specified fields into separate columns. -group, -block, and -subset limit element exploration to selected XML subregions. Processing -cleanup Fix non-ASCII spaces -compress Compress runs of spaces -input Read from file instead of stdin Exploration Argument Hierarchy -pattern Name of record within set -group Use of different argument -block names allows command-line -subset control of nested looping Exploration Constructs Object DateCreated Parent/Child Book/AuthorList Heterogeneous "PubmedArticleSet/*" Nested "*/Taxon" Recursive "**/Gene-commentary" Conditional Execution -if Element [@attribute] required -unless Skip if element matches -and All tests must pass -or Any passing test suffices -else Execute if conditional test failed -position Must be at [first|last] location in list String Constraints -equals String must match exactly -contains Substring must be present -starts-with Substring must be at beginning -ends-with Substring must be at end -is-not String must not match Numeric Constraints -gt Greater than -ge Greater than or equal to -lt Less than -le Less than or equal to -eq Equal to -ne Not equal to Format Customization -ret Override line break between patterns -tab Replace tab character between fields -sep Separator between group members -pfx Prefix to print before group -sfx Suffix to print after group -clr Clear queued tab separator -pfc Preface combines -clr and -pfx -rst Reset -sep, -pfx, and -sfx -def Default placeholder for missing fields -lbl Insert arbitrary text Element Selection -element Print all items that match tag name -first Only print value of first item -last Only print value of last item -NAME Record value in named variable -element Constructs Tag Caption Group Initials,LastName Parent/Child MedlineCitation/PMID Attribute DescriptorName@MajorTopicYN Recursive "**/Gene-commentary_accession" Object Count "#Author" Item Length "%Title" Element Depth "^PMID" Variable "&NAME" Special -element Operations Parent Index "+" XML Subtree "*" Children "$" Attributes "@" Numeric Processing -num Count -len Length -sum Sum -min Minimum -max Maximum -inc Increment -dec Decrement -sub Difference -avg Average -dev Deviation String Processing -encode URL-encode <, >, &, ", and ' characters -upper Convert text to upper-case -lower Convert text to lower-case -title Capitalize initial letters of words Phrase Processing -terms Partition phrase at spaces -words Split at punctuation marks -pairs Adjacent informative words -phrase Experimental index generation -letters Separate individual letters Sequence Coordinates -0-based Zero-Based -1-based One-Based -ucsc-based Half-Open Command Generator -insd Generate INSDSeq extraction commands -insd Argument Order Descriptors INSDSeq_sequence INSDSeq_definition INSDSeq_division Flags [complete|partial] Feature(s) CDS,mRNA Qualifiers INSDFeature_key "#INSDInterval" gene product Miscellaneous -head Print before everything else -tail Print after everything else -hd Print before each record -tl Print after each record Reformatting -format [compact|flush|indent|expand] Modification -filter Object [retain|remove|encode|decode|shrink|expand] [content|cdata|comment|object|attributes|container] Validation -verify Report XML data integrity problems Summary -outline Display outline of XML structure -synopsis Display count of unique XML paths Documentation -help Print this document -examples Examples of EDirect and xtract usage -version Print version number Notes String constraints use case-insensitive comparisons. Numeric constraints and selection arguments use integer values. -num and -len selections are synonyms for Object Count (#) and Item Length (%). -words, -pairs, and -phrase convert to lower case. Examples -pattern DocumentSummary -element Id -first Name Title -pattern "PubmedArticleSet/*" -block Author -sep " " -element Initials,LastName -pattern PubmedArticle -block MeshHeading -if "@MajorTopicYN" -equals Y -sep " / " -element DescriptorName,QualifierName -pattern GenomicInfoType -element ChrAccVer ChrStart ChrStop -pattern Taxon -block "*/Taxon" -unless Rank -equals "no rank" -tab "\n" -element Rank,ScientificName -pattern Entrezgene -block "**/Gene-commentary" -block INSDReference -position 2 -if Author -and Title -if "#Author" -lt 6 -and "%Title" -le 70 -if DateCreated/Year -gt 2005 -if ChrStop -lt ChrStart -if CommonName -contains mouse -if "&ABST" -starts-with "Transposable elements" -if MapLocation -element MapLocation -else -lbl "\-" -min ChrStart,ChrStop -max ExonCount -inc @aaPosition -element @residue -1-based ChrStart -insd CDS gene product protein_id translation -insd complete mat_peptide "%peptide" product peptide -insd CDS INSDInterval_iscomp@value INSDInterval_from INSDInterval_to -filter ExpXml decode content -filter LocationHist remove object ` const xtractInternal = ` ReadBlocks -> SplitPattern => StreamTokens => ParseXML => ProcessQuery -> MergeResults Performance Default Overrides -proc Number of CPU processors used -cons Ratio of parsers to processors -serv Concurrent parser instances -chan Communication channel depth -heap Order restoration heap size -farm Node allocation buffer length -gogc Garbage collection tuning knob Debugging -debug Display run-time parameter summary -empty Flag records with no output -ident Print record index numbers -stats Show processing time for each record -timer Report processing duration and rate -trial Optimize -proc value, requires -input Record Indexing -index Name of element to use for indexing -local Base path for individual XML files Internal Component Performance -chunk ReadBlocks -split ReadBlocks -> SplitPattern -drain ReadBlocks -> SplitPattern -> ConcurrencyChannel -token ReadBlocks -> StreamTokens Documentation -keys Keyboard navigation shortcuts -unix Common Unix commands Sample File Download ftp-cp ftp.ncbi.nlm.nih.gov /entrez/entrezdirect/samples carotene.xml.zip unzip carotene.xml.zip rm carotene.xml.zip Performance Tuning Script XtractTrials() { echo -e "" for tries in {1..5} do xtract -debug -input "$1" -proc "$2" -pattern PubmedArticle -element LastName done echo -e "" } for proc in {1..8} do XtractTrials "carotene.xml" "$proc" | xtract -pattern Trials -lbl "$proc" -avg Rate -dev Rate done Processor Titration Results 1 10455 124 2 18674 684 3 23654 1371 4 30527 521 5 35132 2349 6 40289 907 7 45369 1370 8 47397 1141 Execution Profiling xtract -profile -input carotene.xml -pattern PubmedArticle -element LastName go tool pprof --pdf ./xtract ./cpu.pprof > ./callgraph.pdf ` const xtractExamples = ` Author Frequency esearch -db pubmed -query "rattlesnake phospholipase" | efetch -format docsum | xtract -pattern DocumentSummary -sep "\n" -element Name | sort-uniq-count-rank 39 Marangoni S 31 Toyama MH 26 Soares AM 25 Bon C ... Publications efetch -db pubmed -id 6271474,5685784,4882854,6243420 -format xml | xtract -pattern PubmedArticle -element MedlineCitation/PMID "#Author" \ -block Author -position first -sep " " -element Initials,LastName \ -block Article -element ArticleTitle 6271474 5 MJ Casadaban Tn3: transposition and control. 5685784 2 RK Mortimer Suppressors and suppressible mutations in yeast. 4882854 2 ED Garber Proteins and enzymes as taxonomic tools. 6243420 1 NR Cozzarelli DNA gyrase and the supercoiling of DNA. Formatted Authors efetch -db pubmed -id 1413997,6301692,781293 -format xml | xtract -pattern PubmedArticle -element MedlineCitation/PMID \ -block DateCreated -sep "-" -element Year,Month,Day \ -block Author -sep " " -tab "" \ -element "&COM" Initials,LastName -COM "(, )" 1413997 1992-11-25 RK Mortimer, CR Contopoulou, JS King 6301692 1983-06-17 MA Krasnow, NR Cozzarelli 781293 1976-10-02 MJ Casadaban Medical Subject Headings efetch -db pubmed -id 6092233,2539356,1937004 -format xml | xtract -pattern PubmedArticle -element MedlineCitation/PMID \ -block MeshHeading \ -subset DescriptorName -pfc "\n" -sep "|" -element @MajorTopicYN,DescriptorName \ -subset QualifierName -pfc " / " -sep "|" -element @MajorTopicYN,QualifierName | sed -e 's/N|//g' -e 's/Y|/*/g' 6092233 Base Sequence DNA Restriction Enzymes DNA, Fungal / genetics / *isolation & purification *Genes, Fungal ... Peptide Sequences esearch -db protein -query "conotoxin AND mat_peptide [FKEY]" | efetch -format gpc | xtract -insd complete mat_peptide "%peptide" product peptide | grep -i conotoxin | sort -t $'\t' -u -k 2,2n | head -n 8 ADB43131.1 15 conotoxin Cal 1b LCCKRHHGCHPCGRT ADB43128.1 16 conotoxin Cal 5.1 DPAPCCQHPIETCCRR AIC77105.1 17 conotoxin Lt1.4 GCCSHPACDVNNPDICG ADB43129.1 18 conotoxin Cal 5.2 MIQRSQCCAVKKNCCHVG ADD97803.1 20 conotoxin Cal 1.2 AGCCPTIMYKTGACRTNRCR AIC77085.1 21 conotoxin Bt14.8 NECDNCMRSFCSMIYEKCRLK ADB43125.1 22 conotoxin Cal 14.2 GCPADCPNTCDSSNKCSPGFPG AIC77154.1 23 conotoxin Bt14.19 VREKDCPPHPVPGMHKCVCLKTC Chromosome Locations esearch -db gene -query "calmodulin [PFN] AND mammalia [ORGN]" | efetch -format docsum | xtract -pattern DocumentSummary \ -def "-" -element Id Name MapLocation ScientificName 801 CALM1 14q32.11 Homo sapiens 808 CALM3 19q13.32 Homo sapiens 805 CALM2 2p21 Homo sapiens 24242 Calm1 6q32 Rattus norvegicus 12313 Calm1 12 E Mus musculus 326597 CALM - Bos taurus 50663 Calm2 6q12 Rattus norvegicus 24244 Calm3 1q21 Rattus norvegicus 12315 Calm3 7 9.15 cM Mus musculus 12314 Calm2 17 E4 Mus musculus 617095 CALM1 - Bos taurus 396838 CALM3 6 Sus scrofa ... Gene Regions esearch -db gene -query "DDT [GENE] AND mouse [ORGN]" | efetch -format docsum | xtract -pattern GenomicInfoType -element ChrAccVer ChrStart ChrStop | xargs -n 3 sh -c 'efetch -db nuccore -format gb \ -id "$0" -chr_start "$1" -chr_stop "$2"' LOCUS NC_000076 2142 bp DNA linear CON 09-FEB-2015 DEFINITION Mus musculus strain C57BL/6J chromosome 10, GRCm38.p3 C57BL/6J. ACCESSION NC_000076 REGION: complement(75771233..75773374) GPC_000000783 VERSION NC_000076.6 GI:372099100 ... FEATURES Location/Qualifiers source 1..2142 /organism="Mus musculus" /mol_type="genomic DNA" /strain="C57BL/6J" /db_xref="taxon:10090" /chromosome="10" gene 1..2142 /gene="Ddt" mRNA join(1..159,462..637,1869..2142) /gene="Ddt" /product="D-dopachrome tautomerase" /transcript_id="NM_010027.1" CDS join(52..159,462..637,1869..1941) /gene="Ddt" /codon_start=1 /product="D-dopachrome decarboxylase" /protein_id="NP_034157.1" /translation="MPFVELETNLPASRIPAGLENRLCAATATILDKPEDRVSVTIRP GMTLLMNKSTEPCAHLLVSSIGVVGTAEQNRTHSASFFKFLTEELSLDQDRIVIRFFP ... Taxonomic Names esearch -db taxonomy -query "txid10090 [SBTR] OR camel [COMN]" | efetch -format docsum | xtract -pattern DocumentSummary -if CommonName \ -element Id ScientificName CommonName 57486 Mus musculus molossinus Japanese wild mouse 39442 Mus musculus musculus eastern European house mouse 35531 Mus musculus bactrianus southwestern Asian house mouse 10092 Mus musculus domesticus western European house mouse 10091 Mus musculus castaneus southeastern Asian house mouse 10090 Mus musculus house mouse 9838 Camelus dromedarius Arabian camel 9837 Camelus bactrianus Bactrian camel Structural Similarity esearch -db structure -query "crotalus [ORGN] AND phospholipase A2" | elink -related | efilter -query "archaea [ORGN]" | efetch -format docsum | xtract -pattern DocumentSummary \ -if PdbClass -equals Hydrolase \ -element PdbAcc PdbDescr 3VV2 Crystal Structure Of Complex Form Between S324a-subtilisin And Mutant Tkpro 3VHQ Crystal Structure Of The Ca6 Site Mutant Of Pro-Sa-Subtilisin 2ZWP Crystal Structure Of Ca3 Site Mutant Of Pro-S324a 2ZWO Crystal Structure Of Ca2 Site Mutant Of Pro-S324a ... Multiple Links esearch -db pubmed -query "conotoxin AND dopamine [MAJR]" | elink -target protein -cmd neighbor | xtract -pattern LinkSet -if Link/Id -element IdList/Id Link/Id 23624852 17105332 14657161 27532980 27532978 12944511 31542395 11222635 144922602 Gene Comments esearch -db gene -query "rbcL [GENE] AND maize [ORGN]" | efetch -format xml | xtract -pattern Entrezgene -block "**/Gene-commentary" \ -if Gene-commentary_type@value -equals genomic \ -tab "\n" -element Gene-commentary_accession | sort | uniq NC_001666 X86563 Z11973 Vitamin Biosynthesis esearch -db pubmed -query "tomato lycopene cyclase" | elink -related | elink -target protein | efilter -organism mammals | efetch -format gpc | xtract -pattern INSDSeq -if INSDSeq_definition -contains carotene \ -element INSDSeq_accession-version INSDSeq_definition NP_573480.1 beta,beta-carotene 9',10'-oxygenase [Mus musculus] NP_001156500.1 beta,beta-carotene 15,15'-dioxygenase isoform 2 [Mus musculus] NP_067461.2 beta,beta-carotene 15,15'-dioxygenase isoform 1 [Mus musculus] NP_001297121.1 beta-carotene oxygenase 2 [Mustela putorius furo] AAS20392.1 carotene-9',10'-monooxygenase [Mustela putorius furo] Indexed Fields einfo -db pubmed | xtract -pattern Field \ -if IsDate -equals Y -and IsHidden -equals N \ -pfx "[" -sep "]\t" -element Name,FullName | sort -t $'\t' -k 2f [CDAT] Date - Completion [CRDT] Date - Create [EDAT] Date - Entrez [MHDA] Date - MeSH [MDAT] Date - Modification [PDAT] Date - Publication Author Numbers esearch -db pubmed -query "conotoxin" | efetch -format xml | xtract -pattern PubmedArticle -num Author | sort-uniq-count -n | reorder-columns 2 1 | head -n 15 | xy-plot auth.png 0 11 1 193 2 854 3 844 4 699 5 588 6 439 7 291 8 187 9 124 10 122 11 58 12 33 13 18 900 + | ******** 800 + * ** | * * 700 + * *** | * ** 600 + * * | * *** 500 + * ** | * *** 400 + * ** | * * 300 + * *** | * * 200 + * ****** | * ********* 100 + ** * | * ********** 0 + * ****** +---------+---------+---------+---------+---------+---------+---------+ 0 2 4 6 8 10 12 14 Record Counts echo "diphtheria measles pertussis polio tuberculosis" | xargs -n 1 sh -c 'esearch -db pubmed -query "$0 [MESH]" | efilter -days 365 -datetype PDAT | xtract -pattern ENTREZ_DIRECT -lbl "$0" -element Count' diphtheria 18 measles 166 pertussis 98 polio 75 tuberculosis 1386 Gene Products for sym in HBB DMD TTN ATP7B HFE BRCA2 CFTR PAH PRNP RAG1 do esearch -db gene -query "$sym [GENE] AND human [ORGN]" | efilter -query "alive [PROP]" | efetch -format docsum | xtract -pattern GenomicInfoType \ -element ChrAccVer ChrStart ChrStop | while read acc str stp do efetch -db nuccore -format gbc \ -id "$acc" -chr_start "$str" -chr_stop "$stp" | xtract -insd CDS,mRNA INSDFeature_key "#INSDInterval" \ gene "%transcription" "%translation" \ product transcription translation | grep -i $'\t'"$sym"$'\t' done done NC_000011.10 mRNA 3 HBB 626 hemoglobin, beta ACATTTGCTT... NC_000011.10 CDS 3 HBB 147 hemoglobin subunit beta MVHLTPEEKS... NC_000023.11 mRNA 78 DMD 13805 dystrophin, transcript variant X2 AGGAAGATGA... NC_000023.11 mRNA 77 DMD 13794 dystrophin, transcript variant X6 ACTTTCCCCC... NC_000023.11 mRNA 77 DMD 13800 dystrophin, transcript variant X5 ACTTTCCCCC... NC_000023.11 mRNA 77 DMD 13785 dystrophin, transcript variant X7 ACTTTCCCCC... NC_000023.11 mRNA 74 DMD 13593 dystrophin, transcript variant X8 ACTTTCCCCC... NC_000023.11 mRNA 75 DMD 13625 dystrophin, transcript variant X9 ACTTTCCCCC... ... Genome Range esearch -db gene -query "Homo sapiens [ORGN] AND Y [CHR]" | efilter -status alive | efetch -format docsum | xtract -pattern DocumentSummary -NAME Name -DESC Description \ -block GenomicInfoType -if ChrLoc -equals Y \ -min ChrStart,ChrStop -element "&NAME" "&DESC" | sort -k 1,1n | cut -f 2- | grep -v uncharacterized | between-two-genes ASMT IL3RA IL3RA interleukin 3 receptor subunit alpha SLC25A6 solute carrier family 25 member 6 LINC00106 long intergenic non-protein coding RNA 106 ASMTL-AS1 ASMTL antisense RNA 1 ASMTL acetylserotonin O-methyltransferase-like P2RY8 purinergic receptor P2Y8 AKAP17A A-kinase anchoring protein 17A ASMT acetylserotonin O-methyltransferase 3'UTR Sequences ThreePrimeUTRs() { xtract -pattern INSDSeq -ACC INSDSeq_accession-version -SEQ INSDSeq_sequence \ -block INSDFeature -if INSDFeature_key -equals CDS \ -pfc "\n" -element "&ACC" -rst -last INSDInterval_to -element "&SEQ" | while read acc pos seq do if [ $pos -lt ${#seq} ] then echo -e ">$acc 3'UTR: $((pos+1))..${#seq}" echo "${seq:$pos}" | fold -w 50 elif [ $pos -ge ${#seq} ] then echo -e ">$acc NO 3'UTR" fi done } esearch -db nuccore -query "5.5.1.19 [ECNO]" | efilter -molecule mrna -source refseq | efetch -format gbc | ThreePrimeUTRs >NM_001328461.1 3'UTR: 1737..1871 gatgaatatagagttactgtgttgtaagctaatcatcatactgatgcaag tgcattatcacatttacttctgctgatgattgttcataagattatgagtt agccatttatcaaaaaaaaaaaaaaaaaaaaaaaa >NM_001316759.1 3'UTR: 1628..1690 atccgagtaattcggaatcttgtccaattttatatagcctatattaatac ... Amino Acid Substitutions ApplySNPs() { seq="" last="" while read rsid accn pos res do if [ "$accn" != "$last" ] then insd=$(efetch -db protein -id "$accn" -format gbc < /dev/null) seq=$(echo $insd | xtract -pattern INSDSeq -element INSDSeq_sequence) last=$accn fi pos=$((pos+1)) pfx="" sfx="" echo ">rs$rsid [$accn $res@$pos]" if [ $pos -gt 1 ] then pfx=$(echo ${seq:0:$pos-1}) fi if [ $pos -lt ${#seq} ] then sfx=$(echo ${seq:$pos}) fi echo "$pfx$res$sfx" | fold -w 50 done } esearch -db gene -query "OPN1MW [GENE] AND human [ORGN]" | elink -target snp | efetch -format xml | xtract -pattern Rs -RSID Rs@rsId \ -block FxnSet -if @fxnClass -equals missense \ -sep "." -element "&RSID" @protAcc,@protVer @aaPosition \ -tab "\n" -element @residue | sort -t $'\t' -k 2,2 -k 3,3n -k 4,4 | uniq | ApplySNPs >rs104894915 [NP_000504.1 K@94] maqqwslqrlagrhpqdsyedstqssiftytnsnstrgpfegpnyhiapr wvyhltsvwmifvviasvftnglvlaatmkfkklrhplnwilvKlavadl aetviastisvvnqvygyfvlghpmcvlegytvslcgitglwslaiiswe ... Amino Acid Composition #!/bin/bash -norc abbrev=( Ala Asx Cys Asp Glu Phe Gly His Ile \ Xle Lys Leu Met Asn Pyl Pro Gln Arg \ Ser Thr Sec Val Trp Xxx Tyr Glx ) AminoAcidComp() { local count while read num lttr do idx=$(printf %i "'$lttr'") ofs=$((idx-97)) count[$ofs]="$num" done <<< "$1" for i in {0..25} do echo -e "${abbrev[$i]}\t${count[$i]-0}" done | sort } AminoAcidJoin() { result="" while read acc seq gene do comp="$(echo "$seq" | tr A-Z a-z | sed 's/[^a-z]//g' | fold -w 1 | sort-uniq-count)" current=$(AminoAcidComp "$comp") current=$(echo -e "GENE\t$gene\n$current") if [ -n "$result" ] then result=$(join -t $'\t' <(echo "$result") <(echo "$current")) else result=$current fi done echo "$result" | grep -e "GENE" -e "[1-9]" } ids="NP_001172026,NP_000509,NP_004001,NP_001243779" efetch -db protein -id "$ids" -format gpc | xtract -insd INSDSeq_sequence CDS gene | AminoAcidJoin GENE INS HBB DMD TTN Ala 10 15 210 2084 Arg 5 3 193 1640 Asn 3 6 153 1111 Asp 2 7 185 1720 Cys 6 2 35 513 Gln 7 3 301 942 Glu 8 8 379 3193 Gly 12 13 104 2066 His 2 9 84 478 Ile 2 0 165 2062 Leu 20 18 438 2117 Lys 2 11 282 2943 Met 2 2 79 398 Phe 3 8 77 908 Pro 6 7 130 2517 Ser 5 5 239 2463 Thr 3 7 194 2546 Trp 2 2 67 466 Tyr 4 3 61 999 Val 6 18 186 3184 Word Pair Indexing efetch -db pubmed -id 2981625,12857958 -format xml | xtract -head "" -tail "" -hd "" -tl "" \ -pattern PubmedArticle -if ArticleTitle \ -pfx "" -sfx "" -element MedlineCitation/PMID \ -pfc "" -sfx "" \ -sep "" -pairs ArticleTitle | xtract -pattern Rec -UID Id -block Term -pfc "\n" -element "&UID",Term 2981625 recombination site 2981625 site selection 2981625 tn3 resolvase 2981625 resolvase topological 2981625 topological tests 2981625 tracking mechanism 12857958 chirality sensing 12857958 escherichia coli 12857958 coli topoisomerase 12857958 topoisomerase iv 12857958 type ii 12857958 ii topoisomerases Phrase Searching entrez-phrase-search -db pubmed -field WORD \ selective serotonin reuptake inhibitor + monoamine oxidase inhibitor | efetch -format xml | xtract -pattern PubmedArticle -element MedlineCitation/PMID \ -block Keyword -pfc "\n " -element Keyword 24657329 Antidepressant Organic cation transporter 2 Piperine Uptake 2 24280122 5-HIAA 5-HT 5-HTP 5-hydroxyindoleacetic acid 5-hydroxytryptophan ... Mammalian Sequence Download ftp-ls ftp.ncbi.nlm.nih.gov ncbi-asn1 | grep -e gbmam -e gbpri -e gbrod | xargs ftp-cp ftp.ncbi.nlm.nih.gov ncbi-asn1 Human Subset Extraction for fl in gbpri?.aso.gz gbpri??.aso.gz do run-ncbi-converter asn2all -i "$fl" -a t -b -c -O 9606 -f s > ${fl%.aso.gz}.xml done ` const pubMedArtSample = ` 6301692 1983 06 17 1983 06 17 2007 11 14
0092-8674 32 4 1983 Apr Cell Cell Site-specific relaxation and recombination by the Tn3 resolvase: recognition of the DNA path between oriented res sites. 1313-24 A model in which one subunit of a dimeric resolvase is bound at one res site, while the other searches along adjacent DNA until it encounters the second site, would account for the ability of resolvase to distinguish intramolecular from intermolecular sites, to sense the relative orientation of sites and to produce singly interlinked catenanes. Because resolvase is a type 1 topoisomerase, we infer that it makes the required duplex bDNA breaks of recombination one strand at a time. Krasnow Mark A MA Cozzarelli Nicholas R NR eng GM-07281 GM NIGMS NIH HHS United States Journal Article Research Support, U.S. Gov't, P.H.S.
UNITED STATES Cell 0413066 0092-8674 0 DNA, Bacterial 0 DNA, Superhelical 0 DNA, Viral EC 2.7.7.- Nucleotidyltransferases EC 2.7.7.- Transposases EC 5.99.1.2 DNA Topoisomerases, Type I IM DNA Topoisomerases, Type I metabolism DNA, Bacterial metabolism DNA, Superhelical metabolism DNA, Viral metabolism Models, Genetic Nucleic Acid Conformation Nucleotidyltransferases isolation & purification metabolism Plasmids Recombination, Genetic Repetitive Sequences, Nucleic Acid Simian virus 40 Transposases 1983 4 1 1983 4 1 0 1 1983 4 1 0 0 ppublish 6301692 0092-8674(83)90312-4 ` const insdSeqSample = ` AF480315_1 67 AA linear INV 25-JUL-2016 31-DEC-2003 four-loop conotoxin preproprotein, partial [Conus purpurascens] AAQ05867 AAQ05867.1 gb|AAQ05867.1|AF480315_1 gi|33320307 Conus purpurascens Conus purpurascens Eukaryota; Metazoa; Lophotrochozoa; Mollusca; Gastropoda; Caenogastropoda; Hypsogastropoda; Neogastropoda; Conoidea; Conidae; Conus 1 1..67 Duda,T.F. Jr. Palumbi,S.R. Convergent evolution of venoms and feeding ecologies among polyphyletic piscivorous Conus species Unpublished 2 1..67 Duda,T.F. Jr. Palumbi,S.R. Direct Submission Submitted (04-FEB-2002) Naos Marine Lab, Smithsonian Tropical Research Institute, Apartado 2072, Balboa, Ancon, Panama, Republic of Panama Method: conceptual translation supplied by author. accession AF480315.1 source 1 67 AAQ05867.1 organism Conus purpurascens isolate purpurascens-2c db_xref taxon:41690 clone_lib venom duct cDNA library country Panama note isolated from the Bay of Panama Protein <1..67 1 67 AAQ05867.1 product four-loop conotoxin preproprotein mat_peptide 41..67 41 67 AAQ05867.1 product four-loop conotoxin calculated_mol_wt 3008 peptide PCKKTGRKCFPHQKDCCGRACIITICP CDS 1..67 1 67 AAQ05867.1 coded_by AF480315.1:<1..205 codon_start 2 vvivavlfltacqlitaddsrrtqkhralrsttkratsnrpckktgrkcfphqkdccgraciiticp ` const geneDocSumSample = ` 3581 IL9R interleukin 9 receptor 0 0 X, Y genomic Xq28 and Yq12 CD129, IL-9R interleukin-9 receptor|IL-9 receptor IL9R interleukin 9 receptor Official 300007 X NC_000023.11 155997580 156013016 14 Y NC_000024.10 57184100 57199536 14 5425 The protein encoded by this gene is a cytokine receptor that specifically mediates the biological effects of interleukin 9 (IL9). The functional IL9 receptor complex requires this protein as well as the interleukin 2 receptor, gamma (IL2RG), a common gamma subunit shared by the receptors of many different cytokines. The ligand binding of this receptor leads to the activation of various JAK kinases and STAT proteins, which connect to different biologic responses. This gene is located at the pseudoautosomal regions of X and Y chromosomes. Genetic studies suggested an association of this gene with the development of asthma. Multiple pseudogenes on chromosome 9, 10, 16, and 18 have been described. Alternatively spliced transcript variants have been found for this gene. X 155997580 Homo sapiens human 9606 ` const keyboardShortcuts = ` Command History Ctrl-n Next command Ctrl-p Previous command Move Cursor Forward Ctrl-e To end of line Ctrl-f By one character Esc-f By one argument Move Cursor Backward Ctrl-a To beginning of line Ctrl-b By one character Esc-b By one argument Delete Del Previous character Ctrl-d Next character Ctrl-k To end of line Ctrl-u Entire line Ctrl-w Previous word Esc-Del Previous argument Esc-d Next argument Autocomplete Tab Completes directory or file names Program Control Ctrl-c Quit running program ^x^y Run last command replacing x with y ` const unixCommands = ` Process by Contents sort Sorts lines of text -f Ignore case -n Numeric comparison -r Reverse result order -k Field key (start,stop or first) -u Unique lines with identical keys -b Ignore leading blanks -s Stable sort -t Specify field separator uniq Removes repeated lines -c Count occurrences -i Ignore case -f Ignore first n fields -s Ignore first n characters -d Only output repeated lines -u Only output non-repeated lines grep Matches patterns using regular expressions -i Ignore case -v Invert search -w Search expression as a word -x Search expression as whole line -e Specify individual pattern -c Only count number of matches -n Print line numbers Regular Expressions Characters . Any single character (except newline) \w Alphabetic [A-Za-z], numeric [0-9], or underscore (_) \s Whitespace (space or tab) \ Escapes special characters [] Matches any enclosed characters Positions ^ Beginning of line $ End of line \b Word boundary Repeat Matches ? 0 or 1 * 0 or more + 1 or more {n} Exactly n Escape Sequences \n Line break \t Tab character Modify Contents sed Replaces text strings -e Specify individual expression tr Translates characters -d Delete character rev Reverses characters on line Format Contents column Aligns columns by content width -s Specify field separator -t Create table expand Aligns columns to specified positions -t Tab positions fold Wraps lines at a specific width -w Line width Filter by Position cut Removes parts of lines -c Characters to keep -f Fields to keep -d Specify field separator -s Suppress lines with no delimiters head Prints first lines -n Number of lines tail Prints last lines -n Number of lines Miscellaneous wc Counts words, lines, or characters -c Characters -l Lines -w Words xargs Constructs arguments -n Number of words per batch File Compression tar Archive files -c Create archive -f Name of output file -z Compress archive with gzip gzip Compress file -k Keep original file unzip Decompress .zip archive -p Pipe to stdout gzcat Decompress .gz archive and pipe to stdout Directory and File Navigation cd Changes directory / Root ~ Home . Current .. Parent - Previous ls Lists file names -1 One entry per line -a Show files beginning with dot (.) -l List in long format -R Recursively explore subdirectories -S Sort files by size -t Sort by most recently modified pwd Prints working directory path ` // TYPED CONSTANTS type LevelType int const ( _ LevelType = iota UNIT SUBSET SECTION BLOCK BRANCH GROUP DIVISION PATTERN ) type IndentType int const ( SINGULARITY IndentType = iota COMPACT FLUSH INDENT SUBTREE WRAPPED ) type SideType int const ( _ SideType = iota LEFT RIGHT ) type TagType int const ( NOTAG TagType = iota STARTTAG SELFTAG STOPTAG ATTRIBTAG CONTENTTAG CDATATAG COMMENTTAG OBJECTTAG CONTAINERTAG ISCLOSED ) type OpType int const ( UNSET OpType = iota ELEMENT FIRST LAST ENCODE UPPER LOWER TITLE TERMS WORDS PAIRS PHRASE LETTERS PFX SFX SEP TAB RET LBL CLR PFC RST DEF POSITION IF UNLESS MATCH AVOID AND OR EQUALS CONTAINS STARTSWITH ENDSWITH ISNOT GT GE LT LE EQ NE NUM LEN SUM MIN MAX INC DEC SUB AVG DEV ZEROBASED ONEBASED UCSCBASED ELSE VARIABLE VALUE STAR DOLLAR ATSIGN COUNT LENGTH DEPTH INDEX UNRECOGNIZED ) type ArgumentType int const ( _ ArgumentType = iota EXPLORATION CONDITIONAL EXTRACTION CUSTOMIZATION ) type SpecialType int const ( NOPROCESS SpecialType = iota DOFORMAT DOOUTLINE DOSYNOPSIS DOVERIFY DOFILTER DOCHUNK DOSPLIT DODRAIN DOTOKEN ) type SeqEndType int const ( _ SeqEndType = iota ISSTART ISSTOP ISPOS ) type SequenceType struct { Based int Which SeqEndType } // ARGUMENT MAPS var argTypeIs = map[string]ArgumentType{ "-unit": EXPLORATION, "-Unit": EXPLORATION, "-subset": EXPLORATION, "-Subset": EXPLORATION, "-section": EXPLORATION, "-Section": EXPLORATION, "-block": EXPLORATION, "-Block": EXPLORATION, "-branch": EXPLORATION, "-Branch": EXPLORATION, "-group": EXPLORATION, "-Group": EXPLORATION, "-division": EXPLORATION, "-Division": EXPLORATION, "-pattern": EXPLORATION, "-Pattern": EXPLORATION, "-position": CONDITIONAL, "-if": CONDITIONAL, "-unless": CONDITIONAL, "-match": CONDITIONAL, "-avoid": CONDITIONAL, "-and": CONDITIONAL, "-or": CONDITIONAL, "-equals": CONDITIONAL, "-contains": CONDITIONAL, "-starts-with": CONDITIONAL, "-ends-with": CONDITIONAL, "-is-not": CONDITIONAL, "-gt": CONDITIONAL, "-ge": CONDITIONAL, "-lt": CONDITIONAL, "-le": CONDITIONAL, "-eq": CONDITIONAL, "-ne": CONDITIONAL, "-element": EXTRACTION, "-first": EXTRACTION, "-last": EXTRACTION, "-encode": EXTRACTION, "-upper": EXTRACTION, "-lower": EXTRACTION, "-title": EXTRACTION, "-terms": EXTRACTION, "-words": EXTRACTION, "-pairs": EXTRACTION, "-phrase": EXTRACTION, "-letters": EXTRACTION, "-num": EXTRACTION, "-len": EXTRACTION, "-sum": EXTRACTION, "-min": EXTRACTION, "-max": EXTRACTION, "-inc": EXTRACTION, "-dec": EXTRACTION, "-sub": EXTRACTION, "-avg": EXTRACTION, "-dev": EXTRACTION, "-0-based": EXTRACTION, "-zero-based": EXTRACTION, "-1-based": EXTRACTION, "-one-based": EXTRACTION, "-ucsc": EXTRACTION, "-ucsc-based": EXTRACTION, "-ucsc-coords": EXTRACTION, "-bed-based": EXTRACTION, "-bed-coords": EXTRACTION, "-else": EXTRACTION, "-pfx": CUSTOMIZATION, "-sfx": CUSTOMIZATION, "-sep": CUSTOMIZATION, "-tab": CUSTOMIZATION, "-ret": CUSTOMIZATION, "-lbl": CUSTOMIZATION, "-clr": CUSTOMIZATION, "-pfc": CUSTOMIZATION, "-rst": CUSTOMIZATION, "-def": CUSTOMIZATION, } var opTypeIs = map[string]OpType{ "-element": ELEMENT, "-first": FIRST, "-last": LAST, "-encode": ENCODE, "-upper": UPPER, "-lower": LOWER, "-title": TITLE, "-terms": TERMS, "-words": WORDS, "-pairs": PAIRS, "-phrase": PHRASE, "-letters": LETTERS, "-pfx": PFX, "-sfx": SFX, "-sep": SEP, "-tab": TAB, "-ret": RET, "-lbl": LBL, "-clr": CLR, "-pfc": PFC, "-rst": RST, "-def": DEF, "-position": POSITION, "-if": IF, "-unless": UNLESS, "-match": MATCH, "-avoid": AVOID, "-and": AND, "-or": OR, "-equals": EQUALS, "-contains": CONTAINS, "-starts-with": STARTSWITH, "-ends-with": ENDSWITH, "-is-not": ISNOT, "-gt": GT, "-ge": GE, "-lt": LT, "-le": LE, "-eq": EQ, "-ne": NE, "-num": NUM, "-len": LEN, "-sum": SUM, "-min": MIN, "-max": MAX, "-inc": INC, "-dec": DEC, "-sub": SUB, "-avg": AVG, "-dev": DEV, "-0-based": ZEROBASED, "-zero-based": ZEROBASED, "-1-based": ONEBASED, "-one-based": ONEBASED, "-ucsc": UCSCBASED, "-ucsc-based": UCSCBASED, "-ucsc-coords": UCSCBASED, "-bed-based": UCSCBASED, "-bed-coords": UCSCBASED, "-else": ELSE, } var levelTypeIs = map[string]LevelType{ "-unit": UNIT, "-Unit": UNIT, "-subset": SUBSET, "-Subset": SUBSET, "-section": SECTION, "-Section": SECTION, "-block": BLOCK, "-Block": BLOCK, "-branch": BRANCH, "-Branch": BRANCH, "-group": GROUP, "-Group": GROUP, "-division": DIVISION, "-Division": DIVISION, "-pattern": PATTERN, "-Pattern": PATTERN, } var sequenceTypeIs = map[string]SequenceType{ "INSDSeq:INSDInterval_from": {1, ISSTART}, "INSDSeq:INSDInterval_to": {1, ISSTOP}, "DocumentSummary:ChrStart": {0, ISSTART}, "DocumentSummary:ChrStop": {0, ISSTOP}, "DocumentSummary:Chr_start": {1, ISSTART}, "DocumentSummary:Chr_end": {1, ISSTOP}, "DocumentSummary:Chr_inner_start": {1, ISSTART}, "DocumentSummary:Chr_inner_end": {1, ISSTOP}, "DocumentSummary:Chr_outer_start": {1, ISSTART}, "DocumentSummary:Chr_outer_end": {1, ISSTOP}, "DocumentSummary:start": {1, ISSTART}, "DocumentSummary:stop": {1, ISSTOP}, "DocumentSummary:display_start": {1, ISSTART}, "DocumentSummary:display_stop": {1, ISSTOP}, "Entrezgene:Seq-interval_from": {0, ISSTART}, "Entrezgene:Seq-interval_to": {0, ISSTOP}, "GenomicInfoType:ChrStart": {0, ISSTART}, "GenomicInfoType:ChrStop": {0, ISSTOP}, "Rs:@aaPosition": {0, ISPOS}, "Rs:@asnFrom": {0, ISSTART}, "Rs:@asnTo": {0, ISSTOP}, "Rs:@end": {0, ISSTOP}, "Rs:@leftContigNeighborPos": {0, ISSTART}, "Rs:@physMapInt": {0, ISPOS}, "Rs:@protLoc": {0, ISPOS}, "Rs:@rightContigNeighborPos": {0, ISSTOP}, "Rs:@start": {0, ISSTART}, "Rs:@structLoc": {0, ISPOS}, } var isStopWord = map[string]bool{ "a": true, "about": true, "again": true, "all": true, "almost": true, "also": true, "although": true, "always": true, "among": true, "an": true, "and": true, "another": true, "any": true, "are": true, "as": true, "at": true, "be": true, "because": true, "been": true, "before": true, "being": true, "between": true, "both": true, "but": true, "by": true, "can": true, "could": true, "did": true, "do": true, "does": true, "done": true, "due": true, "during": true, "each": true, "either": true, "enough": true, "especially": true, "etc": true, "for": true, "found": true, "from": true, "further": true, "had": true, "has": true, "have": true, "having": true, "here": true, "how": true, "however": true, "i": true, "if": true, "in": true, "into": true, "is": true, "it": true, "its": true, "itself": true, "just": true, "kg": true, "km": true, "made": true, "mainly": true, "make": true, "may": true, "mg": true, "might": true, "ml": true, "mm": true, "most": true, "mostly": true, "must": true, "nearly": true, "neither": true, "no": true, "nor": true, "obtained": true, "of": true, "often": true, "on": true, "our": true, "overall": true, "perhaps": true, "pmid": true, "quite": true, "rather": true, "really": true, "regarding": true, "seem": true, "seen": true, "several": true, "should": true, "show": true, "showed": true, "shown": true, "shows": true, "significantly": true, "since": true, "so": true, "some": true, "such": true, "than": true, "that": true, "the": true, "their": true, "theirs": true, "them": true, "then": true, "there": true, "therefore": true, "these": true, "they": true, "this": true, "those": true, "through": true, "thus": true, "to": true, "upon": true, "use": true, "used": true, "using": true, "various": true, "very": true, "was": true, "we": true, "were": true, "what": true, "when": true, "which": true, "while": true, "with": true, "within": true, "without": true, "would": true, } // DATA OBJECTS type Tables struct { InBlank [256]bool AltBlank [256]bool InFirst [256]bool InElement [256]bool ChanDepth int FarmSize int Hd string Tl string } type Node struct { Name string Parent string Contents string Attributes string Attribs []string Children *Node Next *Node } type Step struct { Type OpType Value string Parent string Match string Attrib string Wild bool } type Operation struct { Type OpType Value string Stages []*Step } type Block struct { Visit string Parent string Match string Working []string Parsed []string Position string Conditions []*Operation Commands []*Operation Failure []*Operation Subtasks []*Block } // UTILITIES func IsNotJustWhitespace(str string) bool { for _, ch := range str { if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' && ch != '\f' { return true } } return false } func HasAmpOrNotASCII(str string) bool { for _, ch := range str { if ch == '&' || ch > 127 { return true } } return false } func IsAllCapsOrDigits(str string) bool { for _, rune := range str { if !unicode.IsUpper(rune) && !unicode.IsDigit(rune) { return false } } return true } func CompressRunsOfSpaces(str string) string { whiteSpace := false var buffer bytes.Buffer for _, rune := range str { if unicode.IsSpace(rune) { if !whiteSpace { buffer.WriteRune(' ') } whiteSpace = true } else { buffer.WriteRune(rune) whiteSpace = false } } return buffer.String() } func HasFlankingSpace(str string) bool { if str == "" { return false } ch := str[0] if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\f' { return true } strlen := len(str) ch = str[strlen-1] if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' || ch == '\f' { return true } return false } func HasBadSpace(str string) bool { for _, rune := range str { if unicode.IsSpace(rune) && rune != ' ' { return true } } return false } func CleanupBadSpaces(str string) string { var buffer bytes.Buffer for _, rune := range str { if unicode.IsSpace(rune) { buffer.WriteRune(' ') } else { buffer.WriteRune(rune) } } return buffer.String() } func SplitInTwoAt(str, chr string, side SideType) (string, string) { slash := strings.SplitN(str, chr, 2) if len(slash) > 1 { return slash[0], slash[1] } if side == LEFT { return str, "" } return "", str } func ConvertSlash(str string) string { if str == "" { return str } length := len(str) res := make([]byte, length+1, length+1) isSlash := false idx := 0 for _, rune := range str { if isSlash { switch rune { case 'n': // line feed res[idx] = '\n' case 'r': // carriage return res[idx] = '\r' case 't': // horizontal tab res[idx] = '\t' case 'f': // form feed res[idx] = '\f' case 'a': // audible bell from terminal (undocumented) res[idx] = '\x07' default: res[idx] = byte(rune) } idx++ isSlash = false } else if rune == '\\' { isSlash = true } else { res[idx] = byte(rune) idx++ } } res = res[0:idx] return string(res) } func ParseFlag(str string) OpType { op, ok := opTypeIs[str] if ok { return op } if len(str) > 1 && str[0] == '-' && IsAllCapsOrDigits(str[1:]) { return VARIABLE } if len(str) > 0 && str[0] == '-' { return UNRECOGNIZED } return UNSET } // CREATE COMMON DRIVER TABLES // InitTables creates lookup tables to simplify the tokenizer func InitTables() *Tables { tbls := &Tables{} for i := range tbls.InBlank { tbls.InBlank[i] = false } tbls.InBlank[' '] = true tbls.InBlank['\t'] = true tbls.InBlank['\n'] = true tbls.InBlank['\r'] = true tbls.InBlank['\f'] = true // alternative version of InBlank allows newlines to be counted for i := range tbls.AltBlank { tbls.AltBlank[i] = false } tbls.AltBlank[' '] = true tbls.AltBlank['\t'] = true tbls.AltBlank['\r'] = true tbls.AltBlank['\f'] = true // first character of element cannot be a digit, dash, or period for i := range tbls.InFirst { tbls.InFirst[i] = false } for ch := 'A'; ch <= 'Z'; ch++ { tbls.InFirst[ch] = true } for ch := 'a'; ch <= 'z'; ch++ { tbls.InFirst[ch] = true } tbls.InFirst['_'] = true // remaining characters also includes colon for namespace for i := range tbls.InElement { tbls.InElement[i] = false } for ch := 'A'; ch <= 'Z'; ch++ { tbls.InElement[ch] = true } for ch := 'a'; ch <= 'z'; ch++ { tbls.InElement[ch] = true } for ch := '0'; ch <= '9'; ch++ { tbls.InElement[ch] = true } tbls.InElement['_'] = true tbls.InElement['-'] = true tbls.InElement['.'] = true tbls.InElement[':'] = true return tbls } // examine structure of parsed arguments (undocumented) func DebugBlock(blk *Block, depth int) { doIndent := func(indt int) { for i := 1; i < indt; i++ { fmt.Fprintf(os.Stderr, " ") } } doIndent(depth) if blk.Visit != "" { doIndent(depth + 1) fmt.Fprintf(os.Stderr, " %s \n", blk.Visit) } if len(blk.Parsed) > 0 { doIndent(depth + 1) fmt.Fprintf(os.Stderr, "") for _, str := range blk.Parsed { fmt.Fprintf(os.Stderr, " %s", str) } fmt.Fprintf(os.Stderr, " \n") } if len(blk.Subtasks) > 0 { for _, sub := range blk.Subtasks { DebugBlock(sub, depth+1) } } } // PARSE COMMAND-LINE ARGUMENTS // ParseArguments parses nested exploration instruction from command-line arguments func ParseArguments(args []string, pttrn string) *Block { // different names of exploration control arguments allow multiple levels of nested "for" loops in linear command line // (capitalized versions for backward-compatibility with original Perl implementation handling of recursive definitions) var ( lcname = []string{ "", "-unit", "-subset", "-section", "-block", "-branch", "-group", "-division", "-pattern", } ucname = []string{ "", "-Unit", "-Subset", "-Section", "-Block", "-Branch", "-Group", "-Division", "-Pattern", } ) /* xtract -pattern PubmedArticle -element MedlineCitation/PMID \ -block DateCreated -sep "-" -element Year,Month,Day \ -block Author -sep " " -tab "" -element "&COM" Initials,LastName -COM "(, )" PubmedArticle -element MedlineCitation/PMID DateCreated -sep "-" -element Year,Month,Day Author -sep " " -tab "" -element &COM Initials,LastName -COM "(, )" */ // parseCommands recursive definition var parseCommands func(parent *Block, startLevel LevelType) // parseCommands does initial parsing of exploration command structure parseCommands = func(parent *Block, startLevel LevelType) { // function to find next highest level exploration argument findNextLevel := func(args []string, level LevelType) (LevelType, string, string) { if len(args) > 1 { for { if level < UNIT { break } lctag := lcname[level] uctag := ucname[level] for _, txt := range args { if txt == lctag || txt == uctag { return level, lctag, uctag } } level-- } } return 0, "", "" } arguments := parent.Working level, lctag, uctag := findNextLevel(arguments, startLevel) if level < UNIT { // break recursion return } // function to group arguments at a given exploration level subsetCommands := func(args []string) *Block { max := len(args) visit := "" // extract name of object to visit if max > 1 { visit = args[1] args = args[2:] max -= 2 } partition := 0 for cur, str := range args { // record point of next exploration command partition = cur + 1 // skip if not a command if len(str) < 1 || str[0] != '-' { continue } if argTypeIs[str] == EXPLORATION { partition = cur break } } // parse parent/child construct // colon indicates a namespace prefix in any or all of the components prnt, match := SplitInTwoAt(visit, "/", RIGHT) // promote arguments parsed at this level return &Block{Visit: visit, Parent: prnt, Match: match, Parsed: args[0:partition], Working: args[partition:]} } cur := 0 // search for positions of current exploration command for idx, txt := range arguments { if txt == lctag || txt == uctag { if idx == 0 { continue } blk := subsetCommands(arguments[cur:idx]) parseCommands(blk, level-1) parent.Subtasks = append(parent.Subtasks, blk) cur = idx } } if cur < len(arguments) { blk := subsetCommands(arguments[cur:]) parseCommands(blk, level-1) parent.Subtasks = append(parent.Subtasks, blk) } // clear execution arguments from parent after subsetting parent.Working = nil } parseConditionals := func(cmds *Block, arguments []string) []*Operation { max := len(arguments) if max < 1 { return nil } // check for missing condition command txt := arguments[0] if txt != "-if" && txt != "-unless" && txt != "-match" && txt != "-avoid" && txt != "-position" { fmt.Fprintf(os.Stderr, "\nERROR: Missing -if command before '%s'\n", txt) os.Exit(1) } if txt == "-position" && max > 2 { fmt.Fprintf(os.Stderr, "\nERROR: Cannot combine -position with -if or -unless commands\n") os.Exit(1) } // check for missing argument after last condition txt = arguments[max-1] if len(txt) > 0 && txt[0] == '-' { fmt.Fprintf(os.Stderr, "\nERROR: Item missing after %s command\n", txt) os.Exit(1) } cond := make([]*Operation, 0, max) status := UNSET // function to parse conditional clause into execution step parseStep := func(op *Operation, elementColonValue bool) { if op == nil { return } str := op.Value status := ELEMENT // check for pound, percent, or caret character at beginning of name if len(str) > 1 { switch str[0] { case '&': if IsAllCapsOrDigits(str[1:]) { status = VARIABLE str = str[1:] } else if strings.Contains(str, ":") { fmt.Fprintf(os.Stderr, "\nERROR: Unsupported construct '%s', use -if &VARIABLE -equals VALUE instead\n", str) os.Exit(1) } else { fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized variable '%s'\n", str) os.Exit(1) } case '#': status = COUNT str = str[1:] case '%': status = LENGTH str = str[1:] case '^': status = DEPTH str = str[1:] default: } } else if str == "+" { status = INDEX } // parse parent/element@attribute construct // colon indicates a namespace prefix in any or all of the components prnt, match := SplitInTwoAt(str, "/", RIGHT) match, attrib := SplitInTwoAt(match, "@", LEFT) val := "" // leading colon indicates namespace prefix wildcard wildcard := false if strings.HasPrefix(prnt, ":") || strings.HasPrefix(match, ":") || strings.HasPrefix(attrib, ":") { wildcard = true } if elementColonValue { // allow parent/element@attribute:value construct for deprecated -match and -avoid, and for subsequent -and and -or commands match, val = SplitInTwoAt(str, ":", LEFT) prnt, match = SplitInTwoAt(match, "/", RIGHT) match, attrib = SplitInTwoAt(match, "@", LEFT) } tsk := &Step{Type: status, Value: str, Parent: prnt, Match: match, Attrib: attrib, Wild: wildcard} op.Stages = append(op.Stages, tsk) // transform old -match "element:value" to -match element -equals value if val != "" { tsk := &Step{Type: EQUALS, Value: val} op.Stages = append(op.Stages, tsk) } } idx := 0 // conditionals should alternate between command and object/value expectDash := true last := "" var op *Operation // flag to allow element-colon-value for deprecated -match and -avoid commands, otherwise colon is for namespace prefixes elementColonValue := false // parse command strings into operation structure for idx < max { str := arguments[idx] idx++ // conditionals should alternate between command and object/value if expectDash { if len(str) < 1 || str[0] != '-' { fmt.Fprintf(os.Stderr, "\nERROR: Unexpected '%s' argument after '%s'\n", str, last) os.Exit(1) } expectDash = false } else { if len(str) > 0 && str[0] == '-' { fmt.Fprintf(os.Stderr, "\nERROR: Unexpected '%s' command after '%s'\n", str, last) os.Exit(1) } expectDash = true } last = str switch status { case UNSET: status = ParseFlag(str) case POSITION: cmds.Position = str status = UNSET case MATCH, AVOID: elementColonValue = true fallthrough case IF, UNLESS, AND, OR: op = &Operation{Type: status, Value: str} cond = append(cond, op) parseStep(op, elementColonValue) status = UNSET case EQUALS, CONTAINS, STARTSWITH, ENDSWITH, ISNOT: if op != nil { if len(str) > 1 && str[0] == '\\' { // first character may be backslash protecting dash (undocumented) str = str[1:] } tsk := &Step{Type: status, Value: str} op.Stages = append(op.Stages, tsk) op = nil } else { fmt.Fprintf(os.Stderr, "\nERROR: Unexpected adjacent string match constraints\n") os.Exit(1) } status = UNSET case GT, GE, LT, LE, EQ, NE: if op != nil { if len(str) > 1 && str[0] == '\\' { // first character may be backslash protecting minus sign (undocumented) str = str[1:] } if len(str) < 1 { fmt.Fprintf(os.Stderr, "\nERROR: Empty numeric match constraints\n") os.Exit(1) } ch := str[0] if (ch >= '0' && ch <= '9') || ch == '-' || ch == '+' { // literal numeric constant tsk := &Step{Type: status, Value: str} op.Stages = append(op.Stages, tsk) } else { // numeric test allows element as second argument orig := str if ch == '#' || ch == '%' || ch == '^' { // check for pound, percent, or caret character at beginning of element (undocumented) str = str[1:] if len(str) < 1 { fmt.Fprintf(os.Stderr, "\nERROR: Unexpected numeric match constraints\n") os.Exit(1) } ch = str[0] } if (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') { prnt, match := SplitInTwoAt(str, "/", RIGHT) match, attrib := SplitInTwoAt(match, "@", LEFT) wildcard := false if strings.HasPrefix(prnt, ":") || strings.HasPrefix(match, ":") || strings.HasPrefix(attrib, ":") { wildcard = true } tsk := &Step{Type: status, Value: orig, Parent: prnt, Match: match, Attrib: attrib, Wild: wildcard} op.Stages = append(op.Stages, tsk) } else { fmt.Fprintf(os.Stderr, "\nERROR: Unexpected numeric match constraints\n") os.Exit(1) } } op = nil } else { fmt.Fprintf(os.Stderr, "\nERROR: Unexpected adjacent numeric match constraints\n") os.Exit(1) } status = UNSET case UNRECOGNIZED: fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized argument '%s'\n", str) os.Exit(1) default: fmt.Fprintf(os.Stderr, "\nERROR: Unexpected argument '%s'\n", str) os.Exit(1) } } return cond } parseExtractions := func(cmds *Block, arguments []string) []*Operation { max := len(arguments) if max < 1 { return nil } // check for missing -element (or -first, etc.) command txt := arguments[0] if len(txt) < 1 || txt[0] != '-' { fmt.Fprintf(os.Stderr, "\nERROR: Missing -element command before '%s'\n", txt) os.Exit(1) } // check for missing argument after last -element (or -first, etc.) command txt = arguments[max-1] if len(txt) > 0 && txt[0] == '-' { if txt == "-rst" { fmt.Fprintf(os.Stderr, "\nERROR: Unexpected position for %s command\n", txt) os.Exit(1) } else if txt == "-clr" { } else if max < 2 || arguments[max-2] != "-lbl" { fmt.Fprintf(os.Stderr, "\nERROR: Item missing after %s command\n", txt) os.Exit(1) } } comm := make([]*Operation, 0, max) status := UNSET // function to parse next argument nextStatus := func(str string) OpType { status = ParseFlag(str) switch status { case VARIABLE: op := &Operation{Type: status, Value: str[1:]} comm = append(comm, op) status = VALUE case CLR, RST: op := &Operation{Type: status, Value: ""} comm = append(comm, op) status = UNSET case ELEMENT, FIRST, LAST, ENCODE, UPPER, LOWER, TITLE, TERMS, WORDS, PAIRS, PHRASE, LETTERS: case NUM, LEN, SUM, MIN, MAX, INC, DEC, SUB, AVG, DEV, ZEROBASED, ONEBASED, UCSCBASED: case TAB, RET, PFX, SFX, SEP, LBL, PFC, DEF: case UNSET: fmt.Fprintf(os.Stderr, "\nERROR: No -element before '%s'\n", str) os.Exit(1) case UNRECOGNIZED: fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized argument '%s'\n", str) os.Exit(1) default: fmt.Fprintf(os.Stderr, "\nERROR: Misplaced %s command\n", str) os.Exit(1) } return status } // function to parse extraction clause into individual steps parseSteps := func(op *Operation, pttrn string) { if op == nil { return } stat := op.Type str := op.Value // element names combined with commas are treated as a prefix-separator-suffix group comma := strings.Split(str, ",") for _, item := range comma { status := stat // check for special character at beginning of name if len(item) > 1 { switch item[0] { case '&': if IsAllCapsOrDigits(item[1:]) { status = VARIABLE item = item[1:] } else { fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized variable '%s'\n", item) os.Exit(1) } case '#': status = COUNT item = item[1:] case '%': status = LENGTH item = item[1:] case '^': status = DEPTH item = item[1:] case '*': for _, ch := range item { if ch != '*' { break } } status = STAR default: } } else { switch item { case "*": status = STAR case "+": status = INDEX case "$": status = DOLLAR case "@": status = ATSIGN default: } } // parse parent/element@attribute construct // colon indicates a namespace prefix in any or all of the components prnt, match := SplitInTwoAt(item, "/", RIGHT) match, attrib := SplitInTwoAt(match, "@", LEFT) // leading colon indicates namespace prefix wildcard wildcard := false if strings.HasPrefix(prnt, ":") || strings.HasPrefix(match, ":") || strings.HasPrefix(attrib, ":") { wildcard = true } // sequence coordinate adjustments switch status { case ZEROBASED, ONEBASED, UCSCBASED: seq := pttrn + ":" if attrib != "" { seq += "@" seq += attrib } else if match != "" { seq += match } // confirm -0-based or -1-based arguments are known sequence position elements or attributes seqtype, ok := sequenceTypeIs[seq] if !ok { fmt.Fprintf(os.Stderr, "\nERROR: Element '%s' is not suitable for sequence coordinate conversion\n", item) os.Exit(1) } switch status { case ZEROBASED: status = ELEMENT // if 1-based coordinates, decrement to get 0-based value if seqtype.Based == 1 { status = DEC } case ONEBASED: status = ELEMENT // if 0-based coordinates, increment to get 1-based value if seqtype.Based == 0 { status = INC } case UCSCBASED: status = ELEMENT // half-open intervals, start is 0-based, stop is 1-based if seqtype.Based == 0 && seqtype.Which == ISSTOP { status = INC } else if seqtype.Based == 1 && seqtype.Which == ISSTART { status = DEC } default: status = ELEMENT } default: } tsk := &Step{Type: status, Value: item, Parent: prnt, Match: match, Attrib: attrib, Wild: wildcard} op.Stages = append(op.Stages, tsk) } } idx := 0 // parse command strings into operation structure for idx < max { str := arguments[idx] idx++ if argTypeIs[str] == CONDITIONAL { fmt.Fprintf(os.Stderr, "\nERROR: Misplaced %s command\n", str) os.Exit(1) } switch status { case UNSET: status = nextStatus(str) case ELEMENT, FIRST, LAST, ENCODE, UPPER, LOWER, TITLE, TERMS, WORDS, PAIRS, PHRASE, LETTERS, NUM, LEN, SUM, MIN, MAX, INC, DEC, SUB, AVG, DEV, ZEROBASED, ONEBASED, UCSCBASED: for !strings.HasPrefix(str, "-") { // create one operation per argument, even if under a single -element statement op := &Operation{Type: status, Value: str} comm = append(comm, op) parseSteps(op, pttrn) if idx >= max { break } str = arguments[idx] idx++ } status = UNSET if idx < max { status = nextStatus(str) } case TAB, RET, PFX, SFX, SEP, LBL, PFC, DEF: op := &Operation{Type: status, Value: ConvertSlash(str)} comm = append(comm, op) status = UNSET case VARIABLE: op := &Operation{Type: status, Value: str[1:]} comm = append(comm, op) status = VALUE case VALUE: op := &Operation{Type: status, Value: str} comm = append(comm, op) parseSteps(op, pttrn) status = UNSET case UNRECOGNIZED: fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized argument '%s'\n", str) os.Exit(1) default: } } return comm } // parseOperations recursive definition var parseOperations func(parent *Block) // parseOperations converts parsed arguments to operations lists parseOperations = func(parent *Block) { args := parent.Parsed partition := 0 for cur, str := range args { // record junction between conditional and extraction commands partition = cur + 1 // skip if not a command if len(str) < 1 || str[0] != '-' { continue } if argTypeIs[str] != CONDITIONAL { partition = cur break } } // split arguments into conditional tests and extraction or customization commands conditionals := args[0:partition] args = args[partition:] partition = 0 foundElse := false for cur, str := range args { // record junction at -else command partition = cur + 1 // skip if not a command if len(str) < 1 || str[0] != '-' { continue } if str == "-else" { partition = cur foundElse = true break } } extractions := args[0:partition] alternative := args[partition:] if len(alternative) > 0 && alternative[0] == "-else" { alternative = alternative[1:] } // validate argument structure and convert to operations lists parent.Conditions = parseConditionals(parent, conditionals) parent.Commands = parseExtractions(parent, extractions) parent.Failure = parseExtractions(parent, alternative) // reality checks on placement of -else command if foundElse { if len(conditionals) < 1 { fmt.Fprintf(os.Stderr, "\nERROR: Misplaced -else command\n") os.Exit(1) } if len(alternative) < 1 { fmt.Fprintf(os.Stderr, "\nERROR: Misplaced -else command\n") os.Exit(1) } if len(parent.Subtasks) > 0 { fmt.Fprintf(os.Stderr, "\nERROR: Misplaced -else command\n") os.Exit(1) } } for _, sub := range parent.Subtasks { parseOperations(sub) } } // ParseArguments head := &Block{} for _, txt := range args { head.Working = append(head.Working, txt) } // initial parsing of exploration command structure parseCommands(head, PATTERN) if len(head.Subtasks) != 1 { return nil } // skip past empty placeholder head = head.Subtasks[0] // convert command strings to array of operations for faster processing parseOperations(head) // check for no -element or multiple -pattern commands noElement := true numPatterns := 0 for _, txt := range args { if argTypeIs[txt] == EXTRACTION { noElement = false } if txt == "-pattern" || txt == "-Pattern" { numPatterns++ } } if numPatterns < 1 { fmt.Fprintf(os.Stderr, "\nERROR: No -pattern in command-line arguments\n") os.Exit(1) } if numPatterns > 1 { fmt.Fprintf(os.Stderr, "\nERROR: Only one -pattern command is permitted\n") os.Exit(1) } if noElement { fmt.Fprintf(os.Stderr, "\nERROR: No -element statement in argument list\n") os.Exit(1) } return head } // READ XML INPUT FILE INTO SET OF BLOCKS type XMLReader struct { Reader io.Reader Buffer []byte Remainder string Position int64 Delta int Closed bool Docompress bool Docleanup bool } func NewXMLReader(in io.Reader, doCompress, doCleanup bool) *XMLReader { if in == nil { return nil } rdr := &XMLReader{Reader: in, Docompress: doCompress, Docleanup: doCleanup} // 65536 appears to be the maximum number of characters presented to io.Reader when input is piped from stdin // increasing size of buffer when input is from a file does not improve program performance // additional 16384 bytes are reserved for copying previous remainder to start of buffer before next read const XMLBUFSIZE = 65536 + 16384 rdr.Buffer = make([]byte, XMLBUFSIZE) return rdr } // NextBlock reads buffer, concatenates if necessary to place long element content into a single string // all result strings end in > character that is used as a sentinel in subsequent code func (rdr *XMLReader) NextBlock() string { if rdr == nil { return "" } // read one buffer, trim at last > and retain remainder for next call, signal if no > character nextBuffer := func() (string, bool, bool) { if rdr.Closed { return "", false, true } // prepend previous remainder to beginning of buffer m := copy(rdr.Buffer, rdr.Remainder) rdr.Remainder = "" if m > 16384 { // previous remainder is larger than reserved section, write and signal need to continue reading return string(rdr.Buffer[:m]), true, false } // read next block, append behind copied remainder from previous read n, err := rdr.Reader.Read(rdr.Buffer[m:]) // with data piped through stdin, read function may not always return the same number of bytes each time if err != nil { // end of file rdr.Closed = true // do not send final remainder (not terminated by right angle bracket that is used as a sentinel) return "", false, true } // keep track of file offset rdr.Position += int64(rdr.Delta) rdr.Delta = n // slice of actual characters read bufr := rdr.Buffer[:n+m] // look for last > character pos := -1 for pos = len(bufr) - 1; pos >= 0; pos-- { if bufr[pos] == '>' { break } } // trim back to last > character, save remainder for next buffer if pos > -1 { pos++ rdr.Remainder = string(bufr[pos:]) return string(bufr[:pos]), false, false } // no > found, signal need to continue reading long content return string(bufr[:]), true, false } // read next buffer line, cont, closed := nextBuffer() if closed { // no sentinel in remainder at end of file return "" } // if buffer does not end with > character if cont { var buff bytes.Buffer // keep reading long content blocks for { if line != "" { buff.WriteString(line) } if !cont { // last buffer ended with sentinel break } line, cont, closed = nextBuffer() if closed { // no sentinel in multi-block buffer at end of file return "" } } // concatenate blocks line = buff.String() } // trimming spaces here would throw off line tracking // optionally compress/cleanup tags/attributes and contents if rdr.Docompress { line = CompressRunsOfSpaces(line) } if rdr.Docleanup { if HasBadSpace(line) { line = CleanupBadSpaces(line) } } return line } // PARSE XML BLOCK STREAM INTO STRINGS FROM TO // PartitionPattern splits XML input by pattern and sends individual records to a callback func PartitionPattern(pat, star string, rdr *XMLReader, proc func(int, int64, string)) { if pat == "" || rdr == nil || proc == nil { return } type Scanner struct { Pattern string PatLength int CharSkip [256]int } // function to initialize to scanner newScanner := func(pattern string) *Scanner { if pattern == "" { return nil } scr := &Scanner{Pattern: pattern} patlen := len(pattern) scr.PatLength = patlen // position of last character in pattern last := patlen - 1 // initialize bad character displacement table for i := range scr.CharSkip { scr.CharSkip[i] = patlen } for i := 0; i < last; i++ { ch := pattern[i] scr.CharSkip[ch] = last - i } return scr } // function check surroundings of match candidate isAnElement := func(text string, lf, rt, mx int) bool { if (lf >= 0 && text[lf] == '<') || (lf > 0 && text[lf] == '/' && text[lf-1] == '<') { if (rt < mx && (text[rt] == '>' || text[rt] == ' ')) || (rt+1 < mx && text[rt] == '/' && text[rt+1] == '>') { return true } } return false } // modified Boyer-Moore-Horspool search function findNextMatch := func(scr *Scanner, text string, offset int) (int, int, int) { if scr == nil || text == "" { return -1, -1, -1 } // copy values into local variables for speed txtlen := len(text) pattern := scr.Pattern[:] patlen := scr.PatLength max := txtlen - patlen last := patlen - 1 skip := scr.CharSkip[:] i := offset for i <= max { j := last k := i + last for j >= 0 && text[k] == pattern[j] { j-- k-- } // require match candidate to be element name, i.e., , , or if j < 0 && isAnElement(text, i-1, i+patlen, txtlen) { // find positions of flanking brackets lf := i - 1 for lf > 0 && text[lf] != '<' { lf-- } rt := i + patlen for rt < txtlen && text[rt] != '>' { rt++ } return i + 1, lf, rt + 1 } // find character in text above last character in pattern ch := text[i+last] // displacement table can shift pattern by one or more positions i += skip[ch] } return -1, -1, -1 } type PatternType int const ( NOPATTERN PatternType = iota STARTPATTERN SELFPATTERN STOPPATTERN ) // function to find next element with pattern name nextPattern := func(scr *Scanner, text string, pos int) (PatternType, int, int) { if scr == nil || text == "" { return NOPATTERN, 0, 0 } prev := pos for { next, start, stop := findNextMatch(scr, text, prev) if next < 0 { return NOPATTERN, 0, 0 } prev = next + 1 if text[start+1] == '/' { return STOPPATTERN, stop, prev } else if text[stop-2] == '/' { return SELFPATTERN, start, prev } else { return STARTPATTERN, start, prev } } } // -pattern Object construct doNormal := func() { // current depth of -pattern objects level := 0 begin := 0 inPattern := false line := "" var accumulator bytes.Buffer match := NOPATTERN pos := 0 next := 0 offset := int64(0) rec := 0 scr := newScanner(pat) if scr == nil { return } for { begin = 0 next = 0 line = rdr.NextBlock() if line == "" { return } for { match, pos, next = nextPattern(scr, line, next) if match == STARTPATTERN { if level == 0 { inPattern = true begin = pos offset = rdr.Position + int64(pos) } level++ } else if match == STOPPATTERN { level-- if level == 0 { inPattern = false accumulator.WriteString(line[begin:pos]) // read and process one -pattern object at a time str := accumulator.String() if str != "" { rec++ proc(rec, offset, str[:]) } // reset accumulator accumulator.Reset() } } else if match == SELFPATTERN { } else { if inPattern { accumulator.WriteString(line[begin:]) } break } } } } // -pattern Parent/* construct now works with catenated files, but not if components // are recursive or self-closing objects, process those through xtract -format first doStar := func() { // current depth of -pattern objects level := 0 begin := 0 inPattern := false line := "" var accumulator bytes.Buffer match := NOPATTERN pos := 0 next := 0 offset := int64(0) rec := 0 scr := newScanner(pat) if scr == nil { return } last := pat // read to first element for { next = 0 line = rdr.NextBlock() if line == "" { break } match, pos, next = nextPattern(scr, line, next) if match == STARTPATTERN { break } } if match != STARTPATTERN { return } // function to find next element in XML nextElement := func(text string, pos int) string { txtlen := len(text) tag := "" for i := pos; i < txtlen; i++ { if text[i] == '<' { tag = text[i+1:] break } } if tag == "" { return "" } if tag[0] == '/' { if strings.HasPrefix(tag[1:], pat) { //should be at end, want to continue if catenated files return "/" } return "" } for i, ch := range tag { if ch == '>' || ch == ' ' || ch == '/' { return tag[0:i] } } return "" } // read and process heterogeneous objects immediately below parent for { tag := nextElement(line, next) if tag == "" { begin = 0 next = 0 line = rdr.NextBlock() if line == "" { break } tag = nextElement(line, next) } if tag == "" { return } // check for catenated parent set files if tag[0] == '/' { scr = newScanner(pat) if scr == nil { return } last = pat // confirm end just found match, pos, next = nextPattern(scr, line, next) if match != STOPPATTERN { return } // now look for a new start tag for { match, pos, next = nextPattern(scr, line, next) if match == STARTPATTERN { break } next = 0 line = rdr.NextBlock() if line == "" { break } } if match != STARTPATTERN { return } // continue with processing loop continue } if tag != last { scr = newScanner(tag) if scr == nil { return } last = tag } for { match, pos, next = nextPattern(scr, line, next) if match == STARTPATTERN { if level == 0 { inPattern = true begin = pos offset = rdr.Position + int64(pos) } level++ } else if match == STOPPATTERN { level-- if level == 0 { inPattern = false accumulator.WriteString(line[begin:pos]) // read and process one -pattern/* object at a time str := accumulator.String() if str != "" { rec++ proc(rec, offset, str[:]) } // reset accumulator accumulator.Reset() break } } else { if inPattern { accumulator.WriteString(line[begin:]) } begin = 0 next = 0 line = rdr.NextBlock() if line == "" { break } } } } } // call appropriate handler if star == "" { doNormal() } else if star == "*" { doStar() } } // XML VALIDATION AND FORMATTING FUNCTIONS // ProcessXMLStream tokenizes and runs designated operations on an entire XML file func ProcessXMLStream(in *XMLReader, tbls *Tables, args []string, action SpecialType) (int, int) { if in == nil || tbls == nil { return 0, 0 } blockCount := 0 // token parser variables Text := "" Txtlen := 0 Idx := 0 Line := 1 // variables to track comments or CDATA sections that span reader blocks Which := NOTAG SkipTo := "" nextToken := func(idx int) (TagType, string, string, int, int) { if Text == "" { // if buffer is empty, read next block Text = in.NextBlock() Txtlen = len(Text) Idx = 0 idx = 0 blockCount++ } if Text == "" { return ISCLOSED, "", "", Line, 0 } // lookup table array pointers inBlank := &tbls.AltBlank inFirst := &tbls.InFirst inElement := &tbls.InElement text := Text[:] txtlen := Txtlen line := Line if Which != NOTAG && SkipTo != "" { which := Which // previous block ended inside CDATA object or comment start := idx found := strings.Index(text[:], SkipTo) if found < 0 { // no stop signal found in next block // count lines for i := 0; i < txtlen; i++ { if text[i] == '\n' { line++ } } Line = line str := text[:] if HasFlankingSpace(str) { str = strings.TrimSpace(str) } // signal end of current block Text = "" // leave Which and SkipTo values unchanged as another continuation signal // send CDATA or comment contents return which, str[:], "", Line, 0 } // otherwise adjust position past end of skipTo string and return to normal processing idx += found // count lines for i := 0; i < idx; i++ { if text[i] == '\n' { line++ } } Line = line str := text[start:idx] if HasFlankingSpace(str) { str = strings.TrimSpace(str) } idx += len(SkipTo) // clear tracking variables Which = NOTAG SkipTo = "" // send CDATA or comment contents return which, str[:], "", Line, idx } // all blocks end with > character, acts as sentinel to check if past end of text if idx >= txtlen { // signal end of current block, will read next block on next call Text = "" Line = line return NOTAG, "", "", Line, 0 } // skip past leading blanks ch := text[idx] for { for inBlank[ch] { idx++ ch = text[idx] } if ch != '\n' { break } line++ idx++ ch = text[idx] } Line = line start := idx if ch == '<' { // at start of element idx++ ch = text[idx] // check for legal first character of element if inFirst[ch] { // read element name start = idx idx++ ch = text[idx] for inElement[ch] { idx++ ch = text[idx] } str := text[start:idx] switch ch { case '>': // end of element idx++ return STARTTAG, str[:], "", Line, idx case '/': // self-closing element without attributes idx++ ch = text[idx] if ch != '>' { fmt.Fprintf(os.Stderr, "\nSelf-closing element missing right angle bracket\n") } idx++ return SELFTAG, str[:], "", Line, idx case '\n': line++ fallthrough case ' ', '\t', '\r', '\f': // attributes idx++ start = idx ch = text[idx] for { for ch != '<' && ch != '>' && ch != '\n' { idx++ ch = text[idx] } if ch != '\n' { break } line++ idx++ ch = text[idx] } Line = line if ch != '>' { fmt.Fprintf(os.Stderr, "\nAttributes not followed by right angle bracket\n") } if text[idx-1] == '/' { // self-closing atr := text[start : idx-1] idx++ return SELFTAG, str[:], atr[:], Line, idx } atr := text[start:idx] idx++ return STARTTAG, str[:], atr[:], Line, idx default: fmt.Fprintf(os.Stderr, "\nUnexpected punctuation in XML element\n") return STARTTAG, str[:], "", Line, idx } } else { // punctuation character immediately after first angle bracket switch ch { case '/': // at start of end tag idx++ start = idx ch = text[idx] // expect legal first character of element if inFirst[ch] { idx++ ch = text[idx] for inElement[ch] { idx++ ch = text[idx] } str := text[start:idx] if ch != '>' { fmt.Fprintf(os.Stderr, "\nUnexpected characters after end element name\n") } idx++ return STOPTAG, str[:], "", Line, idx } fmt.Fprintf(os.Stderr, "\nUnexpected punctuation in XML element\n") case '?': // skip ?xml and ?processing instructions idx++ ch = text[idx] for ch != '>' { idx++ ch = text[idx] } idx++ return NOTAG, "", "", Line, idx case '!': // skip !DOCTYPE, !comment, and ![CDATA[ idx++ start = idx ch = text[idx] Which := NOTAG SkipTo := "" if ch == '[' && strings.HasPrefix(text[idx:], "[CDATA[") { Which = CDATATAG SkipTo = "]]>" start += 7 } else if ch == '-' && strings.HasPrefix(text[idx:], "--") { Which = COMMENTTAG SkipTo = "-->" start += 2 } if Which != NOTAG && SkipTo != "" { which := Which // CDATA or comment block may contain internal angle brackets found := strings.Index(text[idx:], SkipTo) if found < 0 { // string stops in middle of CDATA or comment // count lines for i := start; i < txtlen; i++ { if text[i] == '\n' { line++ } } Line = line str := text[start:] if HasFlankingSpace(str) { str = strings.TrimSpace(str) } // signal end of current block Text = "" // leave Which and SkipTo values unchanged as another continuation signal // send CDATA or comment contents return which, str[:], "", Line, 0 } // adjust position past end of CDATA or comment idx += found // count lines for i := start; i < idx; i++ { if text[i] == '\n' { line++ } } Line = line str := text[start:idx] if HasFlankingSpace(str) { str = strings.TrimSpace(str) } idx += len(SkipTo) // clear tracking variables Which = NOTAG SkipTo = "" // send CDATA or comment contents return which, str[:], "", Line, idx } // otherwise just skip to next right angle bracket for ch != '>' { if ch == '\n' { line++ } idx++ ch = text[idx] } Line = line idx++ return NOTAG, "", "", Line, idx default: fmt.Fprintf(os.Stderr, "\nUnexpected punctuation in XML element\n") } } } else if ch != '>' { // at start of contents start = idx // find end of contents for { for ch != '<' && ch != '>' && ch != '\n' { idx++ ch = text[idx] } if ch != '\n' { break } line++ idx++ ch = text[idx] } Line = line // trim back past trailing blanks lst := idx - 1 ch = text[lst] for inBlank[ch] && lst > start { lst-- ch = text[lst] } str := text[start : lst+1] return CONTENTTAG, str[:], "", Line, idx } // signal end of current block, will read next block on next call Text = "" Line = line return NOTAG, "", "", Line, 0 } // common output buffer var buffer bytes.Buffer count := 0 // processOutline displays outline of XML structure processOutline := func() { indent := 0 for { tag, name, _, _, idx := nextToken(Idx) Idx = idx switch tag { case STARTTAG: if name == "eSummaryResult" || name == "eLinkResult" || name == "eInfoResult" || name == "PubmedArticleSet" || name == "DocumentSummarySet" || name == "INSDSet" || name == "Entrezgene-Set" || name == "TaxaSet" { break } for i := 0; i < indent; i++ { buffer.WriteString(" ") } buffer.WriteString(name) buffer.WriteString("\n") indent++ case SELFTAG: for i := 0; i < indent; i++ { buffer.WriteString(" ") } buffer.WriteString(name) buffer.WriteString("\n") case STOPTAG: indent-- case NOTAG: case ISCLOSED: txt := buffer.String() if txt != "" { // print final buffer fmt.Fprintf(os.Stdout, "%s", txt) } return default: } count++ if count > 1000 { count = 0 txt := buffer.String() if txt != "" { // print current buffered output fmt.Fprintf(os.Stdout, "%s", txt) } buffer.Reset() } } } // processSynopsis displays paths to XML elements processSynopsis := func() { // synopsisLevel recursive definition var synopsisLevel func(string) bool synopsisLevel = func(parent string) bool { for { tag, name, _, _, idx := nextToken(Idx) Idx = idx switch tag { case STARTTAG: if name == "eSummaryResult" || name == "eLinkResult" || name == "eInfoResult" || name == "PubmedArticleSet" || name == "DocumentSummarySet" || name == "INSDSet" || name == "Entrezgene-Set" || name == "TaxaSet" { break } if parent != "" { buffer.WriteString(parent) buffer.WriteString("/") } buffer.WriteString(name) buffer.WriteString("\n") path := parent if path != "" { path += "/" } path += name if synopsisLevel(path) { return true } case SELFTAG: if parent != "" { buffer.WriteString(parent) buffer.WriteString("/") } buffer.WriteString(name) buffer.WriteString("\n") case STOPTAG: // break recursion return false case NOTAG: case ISCLOSED: txt := buffer.String() if txt != "" { // print final buffer fmt.Fprintf(os.Stdout, "%s", txt) } return true default: } count++ if count > 1000 { count = 0 txt := buffer.String() if txt != "" { // print current buffered output fmt.Fprintf(os.Stdout, "%s", txt) } buffer.Reset() } } } for { // may have concatenated XMLs, loop through all if synopsisLevel("") { return } } } // processVerify checks for well-formed XML processVerify := func() { type VerifyType int const ( _ VerifyType = iota START STOP CHAR OTHER ) // skip past command name args = args[1:] pttrn := "" if len(args) > 0 { pttrn = args[0] args = args[1:] } // if pattern supplied, report maximum nesting depth and record spanning the most blocks (undocumented) maxDepth := 0 depthLine := 0 maxBlocks := 0 blockLine := 0 startLine := 0 // verifyLevel recursive definition var verifyLevel func(string, int) // verify integrity of XML object nesting (well-formed) verifyLevel = func(parent string, level int) { status := START for { // use alternative low-level tokenizer tag, name, _, line, idx := nextToken(Idx) Idx = idx if level > maxDepth { maxDepth = level depthLine = line } switch tag { case STARTTAG: if status == CHAR { fmt.Fprintf(os.Stdout, "<%s> not expected after contents, line %d\n", name, line) } if name == pttrn { blockCount = 1 startLine = line } verifyLevel(name, level+1) // returns here after recursion status = STOP case SELFTAG: status = OTHER case STOPTAG: if name == pttrn { if blockCount > maxBlocks { maxBlocks = blockCount blockLine = startLine } } if parent != name && parent != "" { fmt.Fprintf(os.Stdout, "Expected , found , line %d\n", parent, name, line) } if level < 1 { fmt.Fprintf(os.Stdout, "Unexpected at end of XML, line %d\n", name, line) } // break recursion return case CONTENTTAG: if status != START { fmt.Fprintf(os.Stdout, "Contents not expected before , line %d - status %d\n", parent, line, status) } status = CHAR case CDATATAG, COMMENTTAG: status = OTHER case NOTAG: case ISCLOSED: if level > 0 { fmt.Fprintf(os.Stdout, "Unexpected end of data\n") } return default: status = OTHER } } } verifyLevel("", 0) if pttrn != "" { fmt.Fprintf(os.Stdout, "Maximum nesting (%d levels) at line %d\n", maxDepth, depthLine) fmt.Fprintf(os.Stdout, "Longest pattern (%d blocks) at line %d\n", maxBlocks, blockLine) } } // processFilter modifies XML content, comments, or CDATA processFilter := func() { // skip past command name args = args[1:] max := len(args) if max < 1 { fmt.Fprintf(os.Stderr, "\nERROR: Insufficient command-line arguments supplied to xtract -filter\n") os.Exit(1) } pttrn := args[0] args = args[1:] max-- if max < 2 { fmt.Fprintf(os.Stderr, "\nERROR: No object name supplied to xtract -filter\n") os.Exit(1) } type ActionType int const ( NOACTION ActionType = iota DORETAIN DOREMOVE DOENCODE DODECODE DOSHRINK DOEXPAND ) action := args[0] what := NOACTION switch action { case "retain": what = DORETAIN case "remove": what = DOREMOVE case "encode": what = DOENCODE case "decode": what = DODECODE case "shrink": what = DOSHRINK case "expand": what = DOEXPAND default: fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized action '%s' supplied to xtract -filter\n", action) os.Exit(1) } trget := args[1] which := NOTAG switch trget { case "attribute", "attributes": which = ATTRIBTAG case "content", "contents": which = CONTENTTAG case "cdata", "CDATA": which = CDATATAG case "comment", "comments": which = COMMENTTAG case "object": // object normally retained which = OBJECTTAG case "container": which = CONTAINERTAG default: fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized target '%s' supplied to xtract -filter\n", trget) os.Exit(1) } inPattern := false prevName := "" for { tag, name, attr, _, idx := nextToken(Idx) Idx = idx switch tag { case STARTTAG: prevName = name if name == pttrn { inPattern = true if which == CONTAINERTAG && what == DOREMOVE { continue } } if inPattern && which == OBJECTTAG && what == DOREMOVE { continue } buffer.WriteString("<") buffer.WriteString(name) if attr != "" { if which != ATTRIBTAG || what != DOREMOVE { attr = strings.TrimSpace(attr) attr = CompressRunsOfSpaces(attr) buffer.WriteString(" ") buffer.WriteString(attr) } } buffer.WriteString(">\n") case SELFTAG: if inPattern && which == OBJECTTAG && what == DOREMOVE { continue } buffer.WriteString("<") buffer.WriteString(name) if attr != "" { if which != ATTRIBTAG || what != DOREMOVE { attr = strings.TrimSpace(attr) attr = CompressRunsOfSpaces(attr) buffer.WriteString(" ") buffer.WriteString(attr) } } buffer.WriteString("/>\n") case STOPTAG: if name == pttrn { inPattern = false if which == OBJECTTAG && what == DOREMOVE { continue } if which == CONTAINERTAG && what == DOREMOVE { continue } } if inPattern && which == OBJECTTAG && what == DOREMOVE { continue } buffer.WriteString("\n") case CONTENTTAG: if inPattern && which == OBJECTTAG && what == DOREMOVE { continue } if inPattern && which == CONTENTTAG && what == DOEXPAND { var words []string if strings.Contains(name, "|") { words = strings.FieldsFunc(name, func(c rune) bool { return c == '|' }) } else if strings.Contains(name, ",") { words = strings.FieldsFunc(name, func(c rune) bool { return c == ',' }) } else { words = strings.Fields(name) } between := "" for _, item := range words { max := len(item) for max > 1 { ch := item[max-1] if ch != '.' && ch != ',' && ch != ':' && ch != ';' { break } // trim trailing punctuation item = item[:max-1] // continue checking for runs of punctuation at end max-- } if HasFlankingSpace(item) { item = strings.TrimSpace(item) } if item != "" { if between != "" { buffer.WriteString(between) } buffer.WriteString(item) buffer.WriteString("\n") between = "\n<" + prevName + ">\n" } } continue } if inPattern && which == tag { switch what { case DORETAIN: // default behavior for content - can use -filter X retain content as a no-op case DOREMOVE: continue case DOENCODE: name = html.EscapeString(name) case DODECODE: name = html.UnescapeString(name) case DOSHRINK: name = CompressRunsOfSpaces(name) default: continue } } // content normally printed if HasFlankingSpace(name) { name = strings.TrimSpace(name) } buffer.WriteString(name) buffer.WriteString("\n") case CDATATAG, COMMENTTAG: if inPattern && which == OBJECTTAG && what == DOREMOVE { continue } if inPattern && which == tag { switch what { case DORETAIN: // cdata and comment require explicit retain command case DOREMOVE: continue case DOENCODE: name = html.EscapeString(name) case DODECODE: name = html.UnescapeString(name) case DOSHRINK: name = CompressRunsOfSpaces(name) default: continue } // cdata and comment normally removed if HasFlankingSpace(name) { name = strings.TrimSpace(name) } buffer.WriteString(name) buffer.WriteString("\n") } case NOTAG: case ISCLOSED: txt := buffer.String() if txt != "" { // print final buffer fmt.Fprintf(os.Stdout, "%s", txt) } return default: } count++ if count > 1000 { count = 0 txt := buffer.String() if txt != "" { // print current buffered output fmt.Fprintf(os.Stdout, "%s", txt) } buffer.Reset() } } } // processFormat reformats XML for ease of reading processFormat := func() { // skip past command name args = args[1:] compRecrd := false flushLeft := false wrapAttrs := false ret := "\n" frst := true if len(args) > 0 { switch args[0] { case "compact", "compacted", "compress", "compressed", "terse", "*": // compress to one record per line compRecrd = true ret = "" case "flush", "flushed", "left": // suppress line indentation flushLeft = true case "expand", "expanded", "verbose", "@": // each attribute on its own line wrapAttrs = true case "indent", "indented", "normal": default: fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized option after -format command\n") os.Exit(1) } } type FormatType int const ( NOTSET FormatType = iota START STOP CHAR OTHER ) // array to speed up indentation indentSpaces := []string{ "", " ", " ", " ", " ", " ", " ", " ", " ", " ", } indent := 0 // parent used to detect first start tag, will place in doctype line parent := "" status := NOTSET // delay printing right bracket of start tag to support self-closing tag style needsRightBracket := "" // delay printing start tag if no attributes, suppress empty start-end pair if followed by end justStartName := "" justStartIndent := 0 // function to indent a specified number of spaces doIndent := func(indt int) { if compRecrd || flushLeft { return } i := indt for i > 9 { buffer.WriteString(" ") i -= 10 } if i < 0 { return } buffer.WriteString(indentSpaces[i]) } // function to handle delayed start tag doDelayedName := func() { if needsRightBracket != "" { buffer.WriteString(">") needsRightBracket = "" } if justStartName != "" { doIndent(justStartIndent) buffer.WriteString("<") buffer.WriteString(justStartName) buffer.WriteString(">") justStartName = "" } } closingTag := "" // function to print attributes printAttributes := func(attr string) { attr = strings.TrimSpace(attr) attr = CompressRunsOfSpaces(attr) if wrapAttrs { start := 0 idx := 0 attlen := len(attr) for idx < attlen { ch := attr[idx] if ch == '=' { str := attr[start:idx] buffer.WriteString("\n") doIndent(indent) buffer.WriteString(" ") buffer.WriteString(str) // skip past equal sign and leading double quote idx += 2 start = idx } else if ch == '"' { str := attr[start:idx] buffer.WriteString("=\"") buffer.WriteString(str) buffer.WriteString("\"") // skip past trailing double quote and (possible) space idx += 2 start = idx } else { idx++ } } buffer.WriteString("\n") doIndent(indent) } else { buffer.WriteString(" ") buffer.WriteString(attr) } } for { tag, name, attr, _, idx := nextToken(Idx) Idx = idx switch tag { case STARTTAG: doDelayedName() if status == START { buffer.WriteString(ret) } // remove internal copies of tags if parent != "" && name == parent && indent == 1 { continue } // detect first start tag, print xml and doctype parent if indent == 0 && parent == "" { parent = name buffer.WriteString("\n") buffer.WriteString("\n") // now filtering internal tags, so queue printing of closing tag closingTag = fmt.Sprintf("\n", parent) // already past test, so opening tag will print normally } // check for attributes if attr != "" { doIndent(indent) buffer.WriteString("<") buffer.WriteString(name) printAttributes(attr) needsRightBracket = name } else { justStartName = name justStartIndent = indent } if compRecrd && frst && indent == 0 { frst = false doDelayedName() buffer.WriteString("\n") } indent++ status = START case SELFTAG: doDelayedName() if status == START { buffer.WriteString(ret) } // suppress self-closing tag without attributes attributes if attr != "" { doIndent(indent) buffer.WriteString("<") buffer.WriteString(name) printAttributes(attr) buffer.WriteString("/>") buffer.WriteString(ret) } status = STOP case STOPTAG: // if end immediately follows start, turn into self-closing tag if there were attributes, otherwise suppress empty tag if needsRightBracket != "" { if status == START && name == needsRightBracket { // end immediately follows start, produce self-closing tag buffer.WriteString("/>") buffer.WriteString(ret) needsRightBracket = "" indent-- status = STOP break } buffer.WriteString(">") needsRightBracket = "" } if justStartName != "" { if status == START && name == justStartName { // end immediately follows delayed start with no attributes, suppress justStartName = "" indent-- status = STOP break } doIndent(justStartIndent) buffer.WriteString("<") buffer.WriteString(justStartName) buffer.WriteString(">") justStartName = "" } // remove internal copies of tags if parent != "" && name == parent && indent == 1 { continue } indent-- if status == CHAR { buffer.WriteString("") buffer.WriteString(ret) } else if status == START { buffer.WriteString("") buffer.WriteString(ret) } else { doIndent(indent) buffer.WriteString("") buffer.WriteString(ret) } status = STOP if compRecrd && indent == 1 { buffer.WriteString("\n") } case CONTENTTAG: doDelayedName() if len(name) > 0 && IsNotJustWhitespace(name) { if HasFlankingSpace(name) { name = strings.TrimSpace(name) } buffer.WriteString(name) status = CHAR } case CDATATAG, COMMENTTAG: // ignore case NOTAG: case ISCLOSED: doDelayedName() if closingTag != "" { buffer.WriteString(closingTag) } txt := buffer.String() if txt != "" { // print final buffer fmt.Fprintf(os.Stdout, "%s", txt) } return default: doDelayedName() status = OTHER } count++ if count > 1000 { count = 0 txt := buffer.String() if txt != "" { // print current buffered output fmt.Fprintf(os.Stdout, "%s", txt) } buffer.Reset() } } } // return variables set by performance test functions recordCount := 0 byteCount := 0 // processChunk reads a set of XML blocks processChunk := func() { for { str := in.NextBlock() if str == "" { break } recordCount++ byteCount += len(str) } } // processSplit partitions XML by pattern processSplit := func() { if len(args) > 1 { if args[1] == "-pattern" || args[1] == "-Pattern" { // skip past -split if followed by -pattern args = args[1:] } } if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -split command\n") os.Exit(1) } pat := args[1] PartitionPattern(pat, "", in, func(rec int, ofs int64, str string) { recordCount++ byteCount += len(str) }) } // processDrain partitions XML by pattern and sends them down a channel processDrain := func() { if len(args) > 1 { if args[1] == "-pattern" || args[1] == "-Pattern" { // skip past -drain if followed by -pattern args = args[1:] } } if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -drain command\n") os.Exit(1) } pat := args[1] chn := make(chan string, tbls.ChanDepth) if chn == nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to create output channel\n") os.Exit(1) } sendPatterns := func(pat string, out chan<- string) { defer close(out) PartitionPattern(pat, "", in, func(rec int, ofs int64, str string) { out <- str }) } go sendPatterns(pat, chn) for str := range chn { recordCount++ byteCount += len(str) } } // processToken tokenizes the XML block stream processToken := func() { for { tag, name, attr, _, idx := nextToken(Idx) Idx = idx if tag == ISCLOSED { break } recordCount++ byteCount += len(name) + len(attr) } } // ProcessXMLStream // call specific function switch action { case DOFORMAT: processFormat() case DOOUTLINE: processOutline() case DOSYNOPSIS: processSynopsis() case DOVERIFY: processVerify() case DOFILTER: processFilter() case DOCHUNK: processChunk() case DOSPLIT: processSplit() case DODRAIN: processDrain() case DOTOKEN: processToken() default: } return recordCount, byteCount } // INSDSEQ EXTRACTION COMMAND GENERATOR // e.g., xtract -insd complete mat_peptide "%peptide" product peptide // ProcessINSD generates extraction commands for GenBank/RefSeq records in INSDSet format func ProcessINSD(args []string, isPipe, addDash bool) []string { // legal GenBank / GenPept / RefSeq features features := []string{ "-10_signal", "-35_signal", "3'clip", "3'UTR", "5'clip", "5'UTR", "allele", "assembly_gap", "attenuator", "Bond", "C_region", "CAAT_signal", "CDS", "centromere", "conflict", "D_segment", "D-loop", "enhancer", "exon", "gap", "GC_signal", "gene", "iDNA", "intron", "J_segment", "LTR", "mat_peptide", "misc_binding", "misc_difference", "misc_feature", "misc_recomb", "misc_RNA", "misc_signal", "misc_structure", "mobile_element", "modified_base", "mRNA", "mutation", "N_region", "ncRNA", "old_sequence", "operon", "oriT", "polyA_signal", "polyA_site", "precursor_RNA", "prim_transcript", "primer_bind", "promoter", "propeptide", "protein_bind", "Protein", "RBS", "Region", "regulatory", "rep_origin", "repeat_region", "repeat_unit", "rRNA", "S_region", "satellite", "scRNA", "sig_peptide", "Site", "snoRNA", "snRNA", "source", "stem_loop", "STS", "TATA_signal", "telomere", "terminator", "tmRNA", "transit_peptide", "tRNA", "unsure", "V_region", "V_segment", "variation", } // legal GenBank / GenPept / RefSeq qualifiers qualifiers := []string{ "allele", "altitude", "anticodon", "artificial_location", "bio_material", "bond_type", "bound_moiety", "breed", "calculated_mol_wt", "cell_line", "cell_type", "chloroplast", "chromoplast", "chromosome", "citation", "clone_lib", "clone", "coded_by", "codon_start", "codon", "collected_by", "collection_date", "compare", "cons_splice", "country", "cultivar", "culture_collection", "cyanelle", "db_xref", "derived_from", "dev_stage", "direction", "EC_number", "ecotype", "encodes", "endogenous_virus", "environmental_sample", "estimated_length", "evidence", "exception", "experiment", "focus", "frequency", "function", "gap_type", "gdb_xref", "gene_synonym", "gene", "germline", "haplogroup", "haplotype", "host", "identified_by", "inference", "insertion_seq", "isolate", "isolation_source", "kinetoplast", "lab_host", "label", "lat_lon", "linkage_evidence", "locus_tag", "macronuclear", "map", "mating_type", "metagenome_source", "metagenomic", "mitochondrion", "mobile_element_type", "mobile_element", "mod_base", "mol_type", "name", "nat_host", "ncRNA_class", "non_functional", "note", "number", "old_locus_tag", "operon", "organelle", "organism", "partial", "PCR_conditions", "PCR_primers", "peptide", "phenotype", "plasmid", "pop_variant", "product", "protein_id", "proviral", "pseudo", "pseudogene", "rearranged", "recombination_class", "region_name", "regulatory_class", "replace", "ribosomal_slippage", "rpt_family", "rpt_type", "rpt_unit_range", "rpt_unit_seq", "rpt_unit", "satellite", "segment", "sequenced_mol", "serotype", "serovar", "sex", "site_type", "specific_host", "specimen_voucher", "standard_name", "strain", "structural_class", "sub_clone", "sub_species", "sub_strain", "tag_peptide", "tissue_lib", "tissue_type", "trans_splicing", "transcript_id", "transcription", "transgenic", "transl_except", "transl_table", "translation", "transposon", "type_material", "UniProtKB_evidence", "usedin", "variety", "virion", } // legal INSDSeq XML fields insdtags := []string{ "INSDAltSeqData_items", "INSDAltSeqData", "INSDAltSeqItem_first-accn", "INSDAltSeqItem_gap-comment", "INSDAltSeqItem_gap-length", "INSDAltSeqItem_gap-linkage", "INSDAltSeqItem_gap-type", "INSDAltSeqItem_interval", "INSDAltSeqItem_isgap", "INSDAltSeqItem_isgap@value", "INSDAltSeqItem_last-accn", "INSDAltSeqItem_value", "INSDAltSeqItem", "INSDAuthor", "INSDComment_paragraphs", "INSDComment_type", "INSDComment", "INSDCommentParagraph", "INSDFeature_intervals", "INSDFeature_key", "INSDFeature_location", "INSDFeature_operator", "INSDFeature_partial3", "INSDFeature_partial3@value", "INSDFeature_partial5", "INSDFeature_partial5@value", "INSDFeature_quals", "INSDFeature_xrefs", "INSDFeature", "INSDFeatureSet_annot-source", "INSDFeatureSet_features", "INSDFeatureSet", "INSDInterval_accession", "INSDInterval_from", "INSDInterval_interbp", "INSDInterval_interbp@value", "INSDInterval_iscomp", "INSDInterval_iscomp@value", "INSDInterval_point", "INSDInterval_to", "INSDInterval", "INSDKeyword", "INSDQualifier_name", "INSDQualifier_value", "INSDQualifier", "INSDReference_authors", "INSDReference_consortium", "INSDReference_journal", "INSDReference_position", "INSDReference_pubmed", "INSDReference_reference", "INSDReference_remark", "INSDReference_title", "INSDReference_xref", "INSDReference", "INSDSecondary-accn", "INSDSeq_accession-version", "INSDSeq_alt-seq", "INSDSeq_comment-set", "INSDSeq_comment", "INSDSeq_contig", "INSDSeq_create-date", "INSDSeq_create-release", "INSDSeq_database-reference", "INSDSeq_definition", "INSDSeq_division", "INSDSeq_entry-version", "INSDSeq_feature-set", "INSDSeq_feature-table", "INSDSeq_keywords", "INSDSeq_length", "INSDSeq_locus", "INSDSeq_moltype", "INSDSeq_organism", "INSDSeq_other-seqids", "INSDSeq_primary-accession", "INSDSeq_primary", "INSDSeq_project", "INSDSeq_references", "INSDSeq_secondary-accessions", "INSDSeq_segment", "INSDSeq_sequence", "INSDSeq_source-db", "INSDSeq_source", "INSDSeq_strandedness", "INSDSeq_struc-comments", "INSDSeq_taxonomy", "INSDSeq_topology", "INSDSeq_update-date", "INSDSeq_update-release", "INSDSeq_xrefs", "INSDSeq", "INSDSeqid", "INSDSet", "INSDStrucComment_items", "INSDStrucComment_name", "INSDStrucComment", "INSDStrucCommentItem_tag", "INSDStrucCommentItem_url", "INSDStrucCommentItem_value", "INSDStrucCommentItem", "INSDXref_dbname", "INSDXref_id", "INSDXref", } checkAgainstVocabulary := func(str, objtype string, arry []string) { if str == "" || arry == nil { return } // skip past pound, percent, or caret character at beginning of string if len(str) > 1 { switch str[0] { case '#', '%', '^': str = str[1:] default: } } for _, txt := range arry { if str == txt { return } if strings.ToUpper(str) == strings.ToUpper(txt) { fmt.Fprintf(os.Stderr, "\nERROR: Incorrect capitalization of '%s' %s, change to '%s'\n", str, objtype, txt) os.Exit(1) } } fmt.Fprintf(os.Stderr, "\nERROR: Item '%s' is not a legal -insd %s\n", str, objtype) os.Exit(1) } var acc []string max := len(args) if max < 1 { fmt.Fprintf(os.Stderr, "\nERROR: Insufficient command-line arguments supplied to xtract -insd\n") os.Exit(1) } acc = append(acc, "-pattern", "INSDSeq", "-ACCN", "INSDSeq_accession-version") printAccn := true // collect descriptors if strings.HasPrefix(args[0], "INSD") { if isPipe { acc = append(acc, "-clr", "-pfx", "\\n", "-element", "&ACCN") acc = append(acc, "-group", "INSDSeq", "-sep", "|", "-element") } else { acc = append(acc, "-clr", "-pfx", "\"\\n\"", "-element", "\"&ACCN\"") acc = append(acc, "-group", "INSDSeq", "-sep", "\"|\"", "-element") } printAccn = false for { if len(args) < 1 { return acc } str := args[0] if !strings.HasPrefix(args[0], "INSD") { break } checkAgainstVocabulary(str, "element", insdtags) acc = append(acc, str) args = args[1:] } } else if strings.HasPrefix(strings.ToUpper(args[0]), "INSD") { // report capitalization or vocabulary failure checkAgainstVocabulary(args[0], "element", insdtags) // program should not get to this point, but warn and exit anyway fmt.Fprintf(os.Stderr, "\nERROR: Item '%s' is not a legal -insd %s\n", args[0], "element") os.Exit(1) } // collect qualifiers partial := false complete := false if args[0] == "+" || args[0] == "complete" { complete = true args = args[1:] max-- } else if args[0] == "-" || args[0] == "partial" { partial = true args = args[1:] max-- } if max < 1 { fmt.Fprintf(os.Stderr, "\nERROR: No feature key supplied to xtract -insd\n") os.Exit(1) } acc = append(acc, "-group", "INSDFeature") // limit to designated features feature := args[0] fcmd := "-if" // can specify multiple features separated by plus sign (e.g., CDS+mRNA) or comma (e.g., CDS,mRNA) plus := strings.Split(feature, "+") for _, pls := range plus { comma := strings.Split(pls, ",") for _, cma := range comma { checkAgainstVocabulary(cma, "feature", features) acc = append(acc, fcmd, "INSDFeature_key", "-equals", cma) fcmd = "-or" } } if max < 2 { // still need at least one qualifier even on legal feature fmt.Fprintf(os.Stderr, "\nERROR: Feature '%s' must be followed by at least one qualifier\n", feature) os.Exit(1) } args = args[1:] if complete { acc = append(acc, "-unless", "INSDFeature_partial5", "-or", "INSDFeature_partial3") } else if partial { acc = append(acc, "-if", "INSDFeature_partial5", "-or", "INSDFeature_partial3") } if printAccn { if isPipe { acc = append(acc, "-clr", "-pfx", "\\n", "-element", "&ACCN") } else { acc = append(acc, "-clr", "-pfx", "\"\\n\"", "-element", "\"&ACCN\"") } } for _, str := range args { if strings.HasPrefix(str, "INSD") { checkAgainstVocabulary(str, "element", insdtags) if isPipe { acc = append(acc, "-block", "INSDFeature", "-sep", "|", "-element") } else { acc = append(acc, "-block", "INSDFeature", "-sep", "\"|\"", "-element") } acc = append(acc, str) if addDash { acc = append(acc, "-block", "INSDFeature", "-unless", str) if strings.HasSuffix(str, "@value") { if isPipe { acc = append(acc, "-lbl", "false") } else { acc = append(acc, "-lbl", "\"false\"") } } else { if isPipe { acc = append(acc, "-lbl", "\\-") } else { acc = append(acc, "-lbl", "\"\\-\"") } } } } else if strings.HasPrefix(str, "#INSD") { checkAgainstVocabulary(str, "element", insdtags) if isPipe { acc = append(acc, "-block", "INSDFeature", "-sep", "|", "-element") acc = append(acc, str) } else { acc = append(acc, "-block", "INSDFeature", "-sep", "\"|\"", "-element") ql := fmt.Sprintf("\"%s\"", str) acc = append(acc, ql) } } else if strings.HasPrefix(strings.ToUpper(str), "#INSD") || strings.HasPrefix(strings.ToUpper(str), "#INSD") { // report capitalization or vocabulary failure checkAgainstVocabulary(str, "element", insdtags) } else { acc = append(acc, "-block", "INSDQualifier") checkAgainstVocabulary(str, "qualifier", qualifiers) if len(str) > 2 && str[0] == '%' { acc = append(acc, "-if", "INSDQualifier_name", "-equals", str[1:]) if isPipe { acc = append(acc, "-element", "%INSDQualifier_value") } else { acc = append(acc, "-element", "\"%INSDQualifier_value\"") } if addDash { acc = append(acc, "-block", "INSDFeature", "-unless", "INSDQualifier_name", "-equals", str[1:]) if isPipe { acc = append(acc, "-lbl", "\\-") } else { acc = append(acc, "-lbl", "\"\\-\"") } } } else { acc = append(acc, "-if", "INSDQualifier_name", "-equals", str) acc = append(acc, "-element", "INSDQualifier_value") if addDash { acc = append(acc, "-block", "INSDFeature", "-unless", "INSDQualifier_name", "-equals", str) if isPipe { acc = append(acc, "-lbl", "\\-") } else { acc = append(acc, "-lbl", "\"\\-\"") } } } } } return acc } // HYDRA CITATION MATCHER COMMAND GENERATOR // ProcessHydra generates extraction commands for NCBI's in-house citation matcher (undocumented) func ProcessHydra(isPipe bool) []string { var acc []string // acceptable scores are 0.8 or higher, exact match on "1" rejects low value in scientific notation with minus sign present acc = append(acc, "-pattern", "Id") acc = append(acc, "-if", "@score", "-equals", "1") acc = append(acc, "-or", "@score", "-starts-with", "0.9") acc = append(acc, "-or", "@score", "-starts-with", "0.8") acc = append(acc, "-element", "Id") return acc } // COLLECT AND FORMAT REQUESTED XML VALUES // ParseAttributes is only run if attribute values are requested in element statements func ParseAttributes(attrb string) []string { if attrb == "" { return nil } attlen := len(attrb) // count equal signs num := 0 for i := 0; i < attlen; i++ { if attrb[i] == '=' { num += 2 } } if num < 1 { return nil } // allocate array of proper size arry := make([]string, num) if arry == nil { return nil } start := 0 idx := 0 itm := 0 // place tag and value in successive array slots for idx < attlen && itm < num { ch := attrb[idx] if ch == '=' { // skip past possible leading blanks for start < attlen { ch = attrb[start] if ch == ' ' || ch == '\n' || ch == '\t' || ch == '\r' || ch == '\f' { start++ } else { break } } // = arry[itm] = attrb[start:idx] itm++ // skip past equal sign and leading double quote idx += 2 start = idx } else if ch == '"' { // " arry[itm] = attrb[start:idx] itm++ // skip past trailing double quote and (possible) space idx += 2 start = idx } else { idx++ } } return arry } // ExploreElements returns matching element values to callback func ExploreElements(curr *Node, mask, prnt, match, attrib string, wildcard bool, level int, proc func(string, int)) { if curr == nil || proc == nil { return } // **/Object performs deep exploration of recursive data (*/Object also supported) deep := false if prnt == "**" || prnt == "*" { prnt = "" deep = true } // exploreElements recursive definition var exploreElements func(curr *Node, skip string, lev int) exploreElements = func(curr *Node, skip string, lev int) { if !deep && curr.Name == skip { // do not explore within recursive object return } // wildcard matches any namespace prefix if curr.Name == match || (wildcard && strings.HasPrefix(match, ":") && strings.HasSuffix(curr.Name, match)) || (match == "" && attrib != "") { if prnt == "" || curr.Parent == prnt || (wildcard && strings.HasPrefix(prnt, ":") && strings.HasSuffix(curr.Parent, prnt)) { if attrib != "" { if curr.Attributes != "" && curr.Attribs == nil { // parse attributes on-the-fly if queried curr.Attribs = ParseAttributes(curr.Attributes) } for i := 0; i < len(curr.Attribs)-1; i += 2 { // attributes now parsed into array as [ tag, value, tag, value, tag, value, ... ] if curr.Attribs[i] == attrib || (wildcard && strings.HasPrefix(attrib, ":") && strings.HasSuffix(curr.Attribs[i], attrib)) { proc(curr.Attribs[i+1], level) return } } } else if curr.Contents != "" { str := curr.Contents[:] if HasAmpOrNotASCII(str) { // processing of <, >, &, ", and ' characters is now delayed until element contents is requested str = html.UnescapeString(str) } proc(str, level) return } else if curr.Children != nil { // for XML container object, send empty string to callback to increment count proc("", level) // and continue exploring } else if curr.Attributes != "" { // for self-closing object, indicate presence by sending empty string to callback proc("", level) return } } } for chld := curr.Children; chld != nil; chld = chld.Next { // inner exploration is subject to recursive object exclusion exploreElements(chld, mask, lev+1) } } exploreElements(curr, "", level) } // PrintSubtree supports compression styles selected by -element "*" through "****" func PrintSubtree(node *Node, style IndentType, printAttrs bool, proc func(string)) { if node == nil || proc == nil { return } // WRAPPED is SUBTREE plus each attribute on its own line wrapped := false if style == WRAPPED { style = SUBTREE wrapped = true } // INDENT is offset by two spaces to allow for parent tag, SUBTREE is not offset initial := 1 if style == SUBTREE { style = INDENT initial = 0 } // array to speed up indentation indentSpaces := []string{ "", " ", " ", " ", " ", " ", " ", " ", " ", " ", } // function to indent a specified number of spaces doIndent := func(indt int) { i := indt for i > 9 { proc(" ") i -= 10 } if i < 0 { return } proc(indentSpaces[i]) } // doSubtree recursive definition var doSubtree func(*Node, int) doSubtree = func(curr *Node, depth int) { // suppress if it would be an empty self-closing tag if !IsNotJustWhitespace(curr.Attributes) && curr.Contents == "" && curr.Children == nil { return } if style == INDENT { doIndent(depth) } proc("<") proc(curr.Name) if printAttrs { attr := strings.TrimSpace(curr.Attributes) attr = CompressRunsOfSpaces(attr) if attr != "" { if wrapped { start := 0 idx := 0 attlen := len(attr) for idx < attlen { ch := attr[idx] if ch == '=' { str := attr[start:idx] proc("\n") doIndent(depth) proc(" ") proc(str) // skip past equal sign and leading double quote idx += 2 start = idx } else if ch == '"' { str := attr[start:idx] proc("=\"") proc(str) proc("\"") // skip past trailing double quote and (possible) space idx += 2 start = idx } else { idx++ } } proc("\n") doIndent(depth) } else { proc(" ") proc(attr) } } } // see if suitable for for self-closing tag if curr.Contents == "" && curr.Children == nil { proc("/>") if style != COMPACT { proc("\n") } return } proc(">") if curr.Contents != "" { proc(curr.Contents[:]) } else { if style != COMPACT { proc("\n") } for chld := curr.Children; chld != nil; chld = chld.Next { doSubtree(chld, depth+1) } if style == INDENT { i := depth for i > 9 { proc(" ") i -= 10 } proc(indentSpaces[i]) } } proc("<") proc("/") proc(curr.Name) proc(">") if style != COMPACT { proc("\n") } } doSubtree(node, initial) } // ProcessClause handles comma-separated -element arguments func ProcessClause(curr *Node, stages []*Step, mask, prev, pfx, sfx, sep, def string, status OpType, index, level int, variables map[string]string) (string, bool) { if curr == nil || stages == nil { return "", false } // processElement handles individual -element constructs processElement := func(acc func(string)) { if acc == nil { return } // element names combined with commas are treated as a prefix-separator-suffix group for _, stage := range stages { stat := stage.Type item := stage.Value prnt := stage.Parent match := stage.Match attrib := stage.Attrib wildcard := stage.Wild // exploreElements is a wrapper for ExploreElements, obtaining most arguments as closures exploreElements := func(proc func(string, int)) { ExploreElements(curr, mask, prnt, match, attrib, wildcard, level, proc) } switch stat { case ELEMENT, TERMS, WORDS, PAIRS, PHRASE, LETTERS, VALUE, LEN, SUM, MIN, MAX, SUB, AVG, DEV: exploreElements(func(str string, lvl int) { if str != "" { acc(str) } }) case FIRST: single := "" exploreElements(func(str string, lvl int) { if single == "" { single = str } }) if single != "" { acc(single) } case LAST: single := "" exploreElements(func(str string, lvl int) { single = str }) if single != "" { acc(single) } case ENCODE: exploreElements(func(str string, lvl int) { if str != "" { str = html.EscapeString(str) acc(str) } }) case UPPER: exploreElements(func(str string, lvl int) { if str != "" { str = strings.ToUpper(str) acc(str) } }) case LOWER: exploreElements(func(str string, lvl int) { if str != "" { str = strings.ToLower(str) acc(str) } }) case TITLE: exploreElements(func(str string, lvl int) { if str != "" { str = strings.ToLower(str) str = strings.Title(str) acc(str) } }) case VARIABLE: // use value of stored variable val, ok := variables[match] if ok { acc(val) } case NUM, COUNT: count := 0 exploreElements(func(str string, lvl int) { count++ }) // number of element objects val := strconv.Itoa(count) acc(val) case LENGTH: length := 0 exploreElements(func(str string, lvl int) { length += len(str) }) // length of element strings val := strconv.Itoa(length) acc(val) case DEPTH: exploreElements(func(str string, lvl int) { // depth of each element in scope val := strconv.Itoa(lvl) acc(val) }) case INDEX: // -element "+" prints index of current XML object val := strconv.Itoa(index) acc(val) case INC: // -inc, or component of -0-based, -1-based, or -ucsc-based exploreElements(func(str string, lvl int) { if str != "" { num, err := strconv.Atoi(str) if err == nil { // increment value num++ val := strconv.Itoa(num) acc(val) } } }) case DEC: // -dec, or component of -0-based, -1-based, or -ucsc-based exploreElements(func(str string, lvl int) { if str != "" { num, err := strconv.Atoi(str) if err == nil { // decrement value num-- val := strconv.Itoa(num) acc(val) } } }) case STAR: // -element "*" prints current XML subtree on a single line style := SINGULARITY printAttrs := true for _, ch := range item { if ch == '*' { style++ } else if ch == '@' { printAttrs = false } } if style > WRAPPED { style = WRAPPED } if style < COMPACT { style = COMPACT } var buffer bytes.Buffer PrintSubtree(curr, style, printAttrs, func(str string) { if str != "" { buffer.WriteString(str) } }) txt := buffer.String() if txt != "" { acc(txt) } case DOLLAR: for chld := curr.Children; chld != nil; chld = chld.Next { acc(chld.Name) } case ATSIGN: if curr.Attributes != "" && curr.Attribs == nil { curr.Attribs = ParseAttributes(curr.Attributes) } for i := 0; i < len(curr.Attribs)-1; i += 2 { acc(curr.Attribs[i]) } default: } } } ok := false // format results in buffer var buffer bytes.Buffer buffer.WriteString(prev) buffer.WriteString(pfx) between := "" switch status { case ELEMENT, ENCODE, UPPER, LOWER, TITLE, VALUE, NUM, INC, DEC, ZEROBASED, ONEBASED, UCSCBASED: processElement(func(str string) { if str != "" { ok = true buffer.WriteString(between) buffer.WriteString(str) between = sep } }) case FIRST: single := "" processElement(func(str string) { ok = true if single == "" { single = str } }) if single != "" { buffer.WriteString(between) buffer.WriteString(single) between = sep } case LAST: single := "" processElement(func(str string) { ok = true single = str }) if single != "" { buffer.WriteString(between) buffer.WriteString(single) between = sep } case TERMS: processElement(func(str string) { if str != "" { words := strings.Fields(str) for _, item := range words { max := len(item) for max > 1 { ch := item[max-1] if ch != '.' && ch != ',' && ch != ':' && ch != ';' { break } // trim trailing period, comma, colon, and semicolon item = item[:max-1] // continue checking for runs of punctuation at end max-- } ok = true buffer.WriteString(between) buffer.WriteString(item) between = sep } } }) case WORDS: processElement(func(str string) { if str != "" { words := strings.FieldsFunc(str, func(c rune) bool { return !unicode.IsLetter(c) && !unicode.IsNumber(c) }) for _, item := range words { item = strings.ToLower(item) ok = true buffer.WriteString(between) buffer.WriteString(item) between = sep } } }) case PAIRS: processElement(func(str string) { if str != "" { words := strings.FieldsFunc(str, func(c rune) bool { return !unicode.IsLetter(c) && !unicode.IsNumber(c) }) if len(words) > 1 { past := "" for _, item := range words { item = strings.ToLower(item) if isStopWord[item] { past = "" continue } if past != "" { ok = true buffer.WriteString(between) buffer.WriteString(past + " " + item) between = sep } past = item } } } }) case PHRASE: processElement(func(str string) { if str != "" { words := strings.FieldsFunc(str, func(c rune) bool { return !unicode.IsLetter(c) && !unicode.IsNumber(c) }) buffer.WriteString("") if len(words) > 0 { buffer.WriteString("") for _, item := range words { item = strings.ToLower(item) if isStopWord[item] { continue } ok = true buffer.WriteString("") buffer.WriteString(item) buffer.WriteString("") } buffer.WriteString("") } if len(words) > 1 { buffer.WriteString("") past := "" for _, item := range words { item = strings.ToLower(item) if isStopWord[item] { past = "" continue } if past != "" { ok = true buffer.WriteString("") buffer.WriteString(past + " " + item) buffer.WriteString("") between = sep } past = item } buffer.WriteString("") } buffer.WriteString("") } }) case LETTERS: processElement(func(str string) { if str != "" { for _, ch := range str { ok = true buffer.WriteString(between) buffer.WriteRune(ch) between = sep } } }) case LEN: length := 0 processElement(func(str string) { ok = true length += len(str) }) // length of element strings val := strconv.Itoa(length) buffer.WriteString(between) buffer.WriteString(val) between = sep case SUM: sum := 0 processElement(func(str string) { value, err := strconv.Atoi(str) if err == nil { sum += value ok = true } }) if ok { // sum of element values val := strconv.Itoa(sum) buffer.WriteString(between) buffer.WriteString(val) between = sep } case MIN: min := 0 processElement(func(str string) { value, err := strconv.Atoi(str) if err == nil { if !ok || value < min { min = value } ok = true } }) if ok { // minimum of element values val := strconv.Itoa(min) buffer.WriteString(between) buffer.WriteString(val) between = sep } case MAX: max := 0 processElement(func(str string) { value, err := strconv.Atoi(str) if err == nil { if !ok || value > max { max = value } ok = true } }) if ok { // maximum of element values val := strconv.Itoa(max) buffer.WriteString(between) buffer.WriteString(val) between = sep } case SUB: first := 0 second := 0 count := 0 processElement(func(str string) { value, err := strconv.Atoi(str) if err == nil { count++ if count == 1 { first = value } else if count == 2 { second = value } } }) if count == 2 { // must have exactly 2 elements ok = true // difference of element values val := strconv.Itoa(first - second) buffer.WriteString(between) buffer.WriteString(val) between = sep } case AVG: sum := 0 count := 0 processElement(func(str string) { value, err := strconv.Atoi(str) if err == nil { sum += value count++ ok = true } }) if ok { // average of element values avg := int(float64(sum) / float64(count)) val := strconv.Itoa(avg) buffer.WriteString(between) buffer.WriteString(val) between = sep } case DEV: count := 0 mean := 0.0 m2 := 0.0 processElement(func(str string) { value, err := strconv.Atoi(str) if err == nil { // Welford algorithm for one-pass standard deviation count++ x := float64(value) delta := x - mean mean += delta / float64(count) m2 += delta * (x - mean) } }) if count > 1 { // must have at least 2 elements ok = true // standard deviation of element values vrc := m2 / float64(count-1) dev := int(math.Sqrt(vrc)) val := strconv.Itoa(dev) buffer.WriteString(between) buffer.WriteString(val) between = sep } default: } // use default value if nothing written if !ok && def != "" { ok = true buffer.WriteString(def) } buffer.WriteString(sfx) if !ok { return "", false } txt := buffer.String() return txt, true } // ProcessInstructions performs extraction commands on a subset of XML func ProcessInstructions(commands []*Operation, curr *Node, mask, tab, ret string, index, level int, variables map[string]string, accum func(string)) (string, string) { if accum == nil { return tab, ret } sep := "\t" pfx := "" sfx := "" def := "" col := "\t" lin := "\n" varname := "" // process commands for _, op := range commands { str := op.Value switch op.Type { case ELEMENT, FIRST, LAST, ENCODE, UPPER, LOWER, TITLE, TERMS, WORDS, PAIRS, PHRASE, LETTERS, NUM, LEN, SUM, MIN, MAX, INC, DEC, SUB, AVG, DEV, ZEROBASED, ONEBASED, UCSCBASED: txt, ok := ProcessClause(curr, op.Stages, mask, tab, pfx, sfx, sep, def, op.Type, index, level, variables) if ok { tab = col ret = lin accum(txt) } case TAB: col = str case RET: lin = str case PFX: pfx = str case SFX: sfx = str case SEP: sep = str case LBL: lbl := str accum(tab) accum(lbl) tab = col ret = lin case PFC: // preface clears previous tab and sets prefix in one command pfx = str fallthrough case CLR: // clear previous tab after the fact tab = "" case RST: pfx = "" sfx = "" sep = "\t" def = "" case DEF: def = str case VARIABLE: varname = str case VALUE: length := len(str) if length > 1 && str[0] == '(' && str[length-1] == ')' { // set variable from literal text inside parentheses, e.g., -COM "(, )" variables[varname] = str[1 : length-1] // -if "&VARIABLE" will succeed if set to blank with empty parentheses "()" } else if str == "" { // -if "&VARIABLE" will fail if initialized with empty string "" delete(variables, varname) } else { txt, ok := ProcessClause(curr, op.Stages, mask, "", pfx, sfx, sep, def, op.Type, index, level, variables) if ok { variables[varname] = txt } } varname = "" default: } } return tab, ret } // CONDITIONAL EXECUTION USES -if AND -unless STATEMENT, WITH SUPPORT FOR DEPRECATED -match AND -avoid STATEMENTS // ConditionsAreSatisfied tests a set of conditions to determine if extraction should proceed func ConditionsAreSatisfied(conditions []*Operation, curr *Node, mask string, index, level int, variables map[string]string) bool { if curr == nil { return false } required := 0 observed := 0 forbidden := 0 isMatch := false isAvoid := false // function to test string or numeric constraints testConstraint := func(str string, constraint *Step) bool { if str == "" || constraint == nil { return false } val := constraint.Value stat := constraint.Type switch stat { case EQUALS, CONTAINS, STARTSWITH, ENDSWITH, ISNOT: // substring test on element values str = strings.ToUpper(str) val = strings.ToUpper(val) switch stat { case EQUALS: if str == val { return true } case CONTAINS: if strings.Contains(str, val) { return true } case STARTSWITH: if strings.HasPrefix(str, val) { return true } case ENDSWITH: if strings.HasSuffix(str, val) { return true } case ISNOT: if str != val { return true } default: } case GT, GE, LT, LE, EQ, NE: // second argument of numeric test can be element specifier if constraint.Parent != "" || constraint.Match != "" || constraint.Attrib != "" { ch := val[0] // pound, percent, and caret prefixes supported as potentially useful for data QA (undocumented) switch ch { case '#': count := 0 ExploreElements(curr, mask, constraint.Parent, constraint.Match, constraint.Attrib, constraint.Wild, level, func(stn string, lvl int) { count++ }) val = strconv.Itoa(count) case '%': length := 0 ExploreElements(curr, mask, constraint.Parent, constraint.Match, constraint.Attrib, constraint.Wild, level, func(stn string, lvl int) { if stn != "" { length += len(stn) } }) val = strconv.Itoa(length) case '^': depth := 0 ExploreElements(curr, mask, constraint.Parent, constraint.Match, constraint.Attrib, constraint.Wild, level, func(stn string, lvl int) { depth = lvl }) val = strconv.Itoa(depth) default: ExploreElements(curr, mask, constraint.Parent, constraint.Match, constraint.Attrib, constraint.Wild, level, func(stn string, lvl int) { if stn != "" { _, errz := strconv.Atoi(stn) if errz == nil { val = stn } } }) } } // numeric tests on element values x, errx := strconv.Atoi(str) y, erry := strconv.Atoi(val) // both arguments must resolve to integers if errx != nil || erry != nil { return false } switch stat { case GT: if x > y { return true } case GE: if x >= y { return true } case LT: if x < y { return true } case LE: if x <= y { return true } case EQ: if x == y { return true } case NE: if x != y { return true } default: } default: } return false } // matchFound tests individual conditions matchFound := func(stages []*Step) bool { if stages == nil || len(stages) < 1 { return false } stage := stages[0] var constraint *Step if len(stages) > 1 { constraint = stages[1] } status := stage.Type prnt := stage.Parent match := stage.Match attrib := stage.Attrib wildcard := stage.Wild found := false number := "" // exploreElements is a wrapper for ExploreElements, obtaining most arguments as closures exploreElements := func(proc func(string, int)) { ExploreElements(curr, mask, prnt, match, attrib, wildcard, level, proc) } switch status { case ELEMENT: exploreElements(func(str string, lvl int) { // match to XML container object sends empty string, so do not check for str != "" here // test every selected element individually if value is specified if constraint == nil || testConstraint(str, constraint) { found = true } }) case VARIABLE: // use value of stored variable str, ok := variables[match] if ok { // -if &VARIABLE -equals VALUE is the supported construct if constraint == nil || testConstraint(str, constraint) { found = true } } case COUNT: count := 0 exploreElements(func(str string, lvl int) { count++ found = true }) // number of element objects number = strconv.Itoa(count) case LENGTH: length := 0 exploreElements(func(str string, lvl int) { length += len(str) found = true }) // length of element strings number = strconv.Itoa(length) case DEPTH: depth := 0 exploreElements(func(str string, lvl int) { depth = lvl found = true }) // depth of last element in scope number = strconv.Itoa(depth) case INDEX: // index of explored parent object number = strconv.Itoa(index) found = true default: } if number == "" { return found } if constraint == nil || testConstraint(number, constraint) { return true } return false } // test conditional arguments for _, op := range conditions { switch op.Type { // -if tests for presence of element (deprecated -match can test element:value) case IF, MATCH: // checking for failure here allows for multiple -if [ -and / -or ] clauses if isMatch && observed < required { return false } if isAvoid && forbidden > 0 { return false } required = 0 observed = 0 forbidden = 0 isMatch = true isAvoid = false // continue on to next two cases fallthrough case AND: required++ // continue on to next case fallthrough case OR: if matchFound(op.Stages) { observed++ // record presence of forbidden element if in -unless clause forbidden++ } // -unless tests for absence of element, or presence but with failure of subsequent value test (deprecated -avoid can test element:value) case UNLESS, AVOID: if isMatch && observed < required { return false } if isAvoid && forbidden > 0 { return false } required = 0 observed = 0 forbidden = 0 isMatch = false isAvoid = true if matchFound(op.Stages) { forbidden++ } default: } } if isMatch && observed < required { return false } if isAvoid && forbidden > 0 { return false } return true } // RECURSIVELY PROCESS EXPLORATION COMMANDS AND XML DATA STRUCTURE // ProcessCommands visits XML nodes, performs conditional tests, and executes data extraction instructions func ProcessCommands(cmds *Block, curr *Node, tab, ret string, index, level int, variables map[string]string, accum func(string)) (string, string) { if accum == nil { return tab, ret } prnt := cmds.Parent match := cmds.Match // leading colon indicates namespace prefix wildcard wildcard := false if strings.HasPrefix(prnt, ":") || strings.HasPrefix(match, ":") { wildcard = true } // **/Object performs deep exploration of recursive data deep := false if prnt == "**" { prnt = "*" deep = true } // closure passes local variables to callback, which can modify caller tab and ret values processNode := func(node *Node, idx, lvl int) { // apply -if or -unless tests if ConditionsAreSatisfied(cmds.Conditions, node, match, idx, lvl, variables) { // execute data extraction commands if len(cmds.Commands) > 0 { tab, ret = ProcessInstructions(cmds.Commands, node, match, tab, ret, idx, lvl, variables, accum) } // process sub commands on child node for _, sub := range cmds.Subtasks { tab, ret = ProcessCommands(sub, node, tab, ret, 1, lvl, variables, accum) } } else { // execute commands after -else statement if len(cmds.Failure) > 0 { tab, ret = ProcessInstructions(cmds.Failure, node, match, tab, ret, idx, lvl, variables, accum) } } } // exploreNodes recursive definition var exploreNodes func(*Node, int, int, func(*Node, int, int)) int // exploreNodes visits all nodes that match the selection criteria exploreNodes = func(curr *Node, indx, levl int, proc func(*Node, int, int)) int { if curr == nil || proc == nil { return indx } // match is "*" for heterogeneous data constructs, e.g., -group PubmedArticleSet/* // wildcard matches any namespace prefix if curr.Name == match || match == "*" || (wildcard && strings.HasPrefix(match, ":") && strings.HasSuffix(curr.Name, match)) { if prnt == "" || curr.Parent == prnt || (wildcard && strings.HasPrefix(prnt, ":") && strings.HasSuffix(curr.Parent, prnt)) { proc(curr, indx, levl) indx++ if !deep { // do not explore within recursive object return indx } } } // clearing prnt "*" now allows nested exploration within recursive data, e.g., -pattern Taxon -block */Taxon if prnt == "*" { prnt = "" } // explore child nodes for chld := curr.Children; chld != nil; chld = chld.Next { indx = exploreNodes(chld, indx, levl+1, proc) } return indx } // apply -position test if cmds.Position == "" { exploreNodes(curr, index, level, processNode) } else { var single *Node lev := 0 ind := 0 if cmds.Position == "first" { exploreNodes(curr, index, level, func(node *Node, idx, lvl int) { if single == nil { single = node ind = idx lev = lvl } }) } else if cmds.Position == "last" { exploreNodes(curr, index, level, func(node *Node, idx, lvl int) { single = node ind = idx lev = lvl }) } else { // use numeric position number, err := strconv.Atoi(cmds.Position) if err == nil { pos := 0 exploreNodes(curr, index, level, func(node *Node, idx, lvl int) { pos++ if pos == number { single = node ind = idx lev = lvl } }) } else { fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized position '%s'\n", cmds.Position) os.Exit(1) } } if single != nil { processNode(single, ind, lev) } } return tab, ret } // PROCESS ONE XML COMPONENT RECORD // ParseXML combines the tokenizer and parser to work on a single partitioned XML string func ParseXML(Text, parent string, tbls *Tables) (*Node, bool) { if Text == "" || tbls == nil { return nil, false } // node farm variables FarmPos := 0 FarmMax := tbls.FarmSize FarmItems := make([]Node, FarmMax) // function to allocate multiple nodes in a large array for memory management efficiency nextNode := func(strt, attr, prnt string) *Node { // if farm array slots used up, allocate new array if FarmPos >= FarmMax { FarmItems = make([]Node, FarmMax) FarmPos = 0 } if FarmItems == nil { return nil } // take node from next available slot in farm array node := &FarmItems[FarmPos] node.Name = strt[:] node.Attributes = attr[:] node.Parent = prnt[:] FarmPos++ return node } // token parser variables Txtlen := len(Text) Idx := 0 // function to get next XML token nextToken := func(idx int) (TagType, string, string, int) { // lookup table array pointers inBlank := &tbls.InBlank inFirst := &tbls.InFirst inElement := &tbls.InElement text := Text[:] txtlen := Txtlen // XML string ends with > character, acts as sentinel to check if past end of text if idx >= txtlen { // signal end of XML string return ISCLOSED, "", "", 0 } // skip past leading blanks ch := text[idx] for inBlank[ch] { idx++ ch = text[idx] } start := idx if ch == '<' { // at start of element idx++ ch = text[idx] // check for legal first character of element if inFirst[ch] { // read element name start = idx idx++ ch = text[idx] for inElement[ch] { idx++ ch = text[idx] } str := text[start:idx] switch ch { case '>': // end of element idx++ return STARTTAG, str[:], "", idx case '/': // self-closing element without attributes idx++ ch = text[idx] if ch != '>' { fmt.Fprintf(os.Stderr, "\nSelf-closing element missing right angle bracket\n") } idx++ return SELFTAG, str[:], "", idx case ' ', '\t', '\n', '\r', '\f': // attributes idx++ start = idx ch = text[idx] for ch != '<' && ch != '>' { idx++ ch = text[idx] } if ch != '>' { fmt.Fprintf(os.Stderr, "\nAttributes not followed by right angle bracket\n") } if text[idx-1] == '/' { // self-closing atr := text[start : idx-1] idx++ return SELFTAG, str[:], atr[:], idx } atr := text[start:idx] idx++ return STARTTAG, str[:], atr[:], idx default: fmt.Fprintf(os.Stderr, "\nUnexpected punctuation in XML element\n") return STARTTAG, str[:], "", idx } } else { // punctuation character immediately after first angle bracket switch ch { case '/': // at start of end tag idx++ start = idx ch = text[idx] // expect legal first character of element if inFirst[ch] { idx++ ch = text[idx] for inElement[ch] { idx++ ch = text[idx] } str := text[start:idx] if ch != '>' { fmt.Fprintf(os.Stderr, "\nUnexpected characters after end element name\n") } idx++ return STOPTAG, str[:], "", idx } fmt.Fprintf(os.Stderr, "\nUnexpected punctuation in XML element\n") case '?': // skip ?xml and ?processing instructions idx++ ch = text[idx] for ch != '>' { idx++ ch = text[idx] } idx++ case '!': // skip !DOCTYPE, !comment, and ![CDATA[ idx++ start = idx ch = text[idx] which := NOTAG skipTo := "" if ch == '[' && strings.HasPrefix(text[idx:], "[CDATA[") { which = CDATATAG skipTo = "]]>" start += 7 } else if ch == '-' && strings.HasPrefix(text[idx:], "--") { which = COMMENTTAG skipTo = "-->" start += 2 } if which != NOTAG && skipTo != "" { // CDATA or comment block may contain internal angle brackets found := strings.Index(text[idx:], skipTo) if found < 0 { // string stops in middle of CDATA or comment return ISCLOSED, "", "", idx } // adjust position past end of CDATA or comment idx += found + len(skipTo) } else { // otherwise just skip to next right angle bracket for ch != '>' { idx++ ch = text[idx] } idx++ } default: fmt.Fprintf(os.Stderr, "\nUnexpected punctuation in XML element\n") } } } else if ch != '>' { // at start of contents start = idx // find end of contents for ch != '<' && ch != '>' { idx++ ch = text[idx] } // trim back past trailing blanks lst := idx - 1 ch = text[lst] for inBlank[ch] && lst > start { lst-- ch = text[lst] } str := text[start : lst+1] return CONTENTTAG, str[:], "", idx } return NOTAG, "", "", idx } // parseLevel recursive definition var parseLevel func(string, string, string) (*Node, bool) parseLevel = func(strt, attr, prnt string) (*Node, bool) { ok := true // obtain next node from farm node := nextNode(strt, attr, prnt) if node == nil { return nil, false } var lastNode *Node for { tag, name, attr, idx := nextToken(Idx) if tag == ISCLOSED { break } Idx = idx switch tag { case STARTTAG: // read sub tree obj, ok := parseLevel(name, attr, node.Name) if !ok { break } // adding next child to end of linked list gives better performance than appending to slice of nodes if node.Children == nil { node.Children = obj } if lastNode != nil { lastNode.Next = obj } lastNode = obj case STOPTAG: // pop out of recursive call return node, ok case CONTENTTAG: node.Contents = name case SELFTAG: if attr == "" { // ignore if self-closing tag has no attributes continue } // self-closing tag has no contents, just create child node obj := nextNode(name, attr, node.Name) if node.Children == nil { node.Children = obj } if lastNode != nil { lastNode.Next = obj } lastNode = obj // continue on same level default: } } return node, ok } for { tag, name, attr, idx := nextToken(Idx) if tag == ISCLOSED { break } Idx = idx if tag != STARTTAG { continue } // call recursive function from beginning of XML return parseLevel(name, attr, parent) } return nil, false } // ProcessQuery calls XML combined tokenizer parser on a partitioned string func ProcessQuery(text, parent string, index int, cmds *Block, tbls *Tables) string { if text == "" || cmds == nil || tbls == nil { return "" } // exit from function will collect garbage of node structure for current XML object pat, ok := ParseXML(text, parent, tbls) if !ok { return "" } // exit from function will also free map of recorded variables for current -pattern variables := make(map[string]string) var buffer bytes.Buffer ok = false if tbls.Hd != "" { buffer.WriteString(tbls.Hd[:]) } // start processing at top of command tree and top of XML subregion selected by -pattern _, ret := ProcessCommands(cmds, pat, "", "", index, 1, variables, func(str string) { if str != "" { ok = true buffer.WriteString(str) } }) if tbls.Tl != "" { buffer.WriteString(tbls.Tl[:]) } if ret != "" { ok = true buffer.WriteString(ret) } txt := buffer.String() // remove leading newline (-insd -pfx artifact) if txt != "" && txt[0] == '\n' { txt = txt[1:] } if !ok { return "" } // return consolidated result string return txt } // UNSHUFFLER USES HEAP TO RESTORE OUTPUT OF MULTIPLE CONSUMERS TO ORIGINAL RECORD ORDER type Extract struct { Index int Text string } type ExtractHeap []Extract // methods that satisfy heap.Interface func (h ExtractHeap) Len() int { return len(h) } func (h ExtractHeap) Less(i, j int) bool { return h[i].Index < h[j].Index } func (h ExtractHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } func (h *ExtractHeap) Push(x interface{}) { *h = append(*h, x.(Extract)) } func (h *ExtractHeap) Pop() interface{} { old := *h n := len(old) x := old[n-1] *h = old[0 : n-1] return x } // CONCURRENT CONSUMER GOROUTINES PARSE AND PROCESS PARTITIONED XML OBJECTS // ReadBlocks -> SplitPattern => StreamTokens => ParseXML => ProcessQuery -> MergeResults // XMLProducer sends partitioned XML strings through channel func XMLProducer(pat, star string, rdr *XMLReader, out chan<- Extract) { // close channel when all records have been processed, so consumers can range over channel defer close(out) // partition all input by pattern and send XML substring to available consumer through channel PartitionPattern(pat, star, rdr, func(rec int, ofs int64, str string) { out <- Extract{rec, str} }) } func CreateProducer(pat, star string, rdr *XMLReader, tbls *Tables) <-chan Extract { if tbls == nil { return nil } out := make(chan Extract, tbls.ChanDepth) if out == nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to create producer channel\n") os.Exit(1) } // launch single producer goroutine go XMLProducer(pat, star, rdr, out) return out } // XMLConsumer reads partitioned XML from channel and calls parser for processing func XMLConsumer(cmds *Block, tbls *Tables, parent string, wg *sync.WaitGroup, inp <-chan Extract, out chan<- Extract) { // report when this consumer has no more records to process defer wg.Done() // read partitioned XML from producer channel for ext := range inp { idx := ext.Index text := ext.Text if text == "" { // should never see empty input data out <- Extract{idx, text} continue } str := ProcessQuery(text[:], parent, idx, cmds, tbls) // send even if empty to get all record counts for reordering out <- Extract{idx, str} } } func CreateConsumers(cmds *Block, tbls *Tables, parent string, numServers int, inp <-chan Extract) <-chan Extract { if tbls == nil { return nil } out := make(chan Extract, tbls.ChanDepth) if out == nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to create consumer channel\n") os.Exit(1) } var wg sync.WaitGroup // launch multiple consumer goroutines for i := 0; i < numServers; i++ { wg.Add(1) go XMLConsumer(cmds, tbls, parent, &wg, inp, out) } // launch separate anonymous goroutine to wait until all consumers are done, then close single output channel, so unshuffler can range over channel go func() { wg.Wait() close(out) }() return out } // MAIN FUNCTION // e.g., xtract -pattern PubmedArticle -element MedlineCitation/PMID -block Author -sep " " -element Initials,LastName func main() { // skip past executable name args := os.Args[1:] if len(args) < 1 { fmt.Fprintf(os.Stderr, "\nERROR: No command-line arguments supplied to xtract\n") os.Exit(1) } // CONCURRENCY, CLEANUP, AND DEBUGGING FLAGS // do these first because -defcpu and -maxcpu can be sent from wrapper before other arguments ncpu := runtime.NumCPU() if ncpu < 1 { ncpu = 1 } // wrapper can limit maximum number of processors to use (undocumented) maxProcs := ncpu defProcs := 0 // concurrent performance tuning parameters, can be overridden by -proc and -cons numProcs := 0 serverRatio := 4 // number of servers usually calculated by -cons server ratio, but can be overridden by -serv numServers := 0 // number of channels usually equals number of servers, but can be overridden by -chan chanDepth := 0 // miscellaneous tuning parameters heapSize := 16 farmSize := 64 // garbage collector control can be set by environment variable or default value with -gogc 0 goGc := 600 // XML data cleanup doCompress := false doCleanup := false // read data from file instead of stdin fileName := "" // debugging dbug := false mpty := false idnt := false stts := false timr := false // profiling prfl := false // element to use as local data index indx := "" // path for local data indexed as trie local := "" // alternative source of sample record, processed a designated number of times, looping for each -proc from 1 to nCPU (undocumented) testCount := 0 testType := "" testString := "" // repeat the specified extraction 5 times for each -proc from 1 to nCPU trial := false // function to get numeric value getNumericArg := func(name string, zer, min, max int) int { if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: %s is missing\n", name) os.Exit(1) } value, err := strconv.Atoi(args[1]) if err != nil { fmt.Fprintf(os.Stderr, "\nERROR: %s (%s) is not an integer\n", name, args[1]) os.Exit(1) } // skip past first of two arguments args = args[1:] // special case for argument value of 0 if value < 1 { return zer } // limit value to between specified minimum and maximum if value < min { return min } if value > max { return max } return value } inSwitch := true // get concurrency, cleanup, and debugging flags in any order for { inSwitch = true switch args[0] { // concurrency override arguments can be passed in by local wrapper script (undocumented) case "-maxcpu": maxProcs = getNumericArg("Maximum number of processors", 1, 1, ncpu) case "-defcpu": defProcs = getNumericArg("Default number of processors", ncpu, 1, ncpu) // performance tuning flags case "-proc": numProcs = getNumericArg("Number of processors", ncpu, 1, ncpu) case "-cons": serverRatio = getNumericArg("Parser to processor ratio", 4, 1, 32) case "-serv": numServers = getNumericArg("Concurrent parser count", 0, ncpu, 128) case "-chan": chanDepth = getNumericArg("Communication channel depth", 0, ncpu, 128) case "-heap": heapSize = getNumericArg("Unshuffler heap size", 8, 8, 64) case "-farm": farmSize = getNumericArg("Node buffer length", 4, 4, 2048) case "-gogc": goGc = getNumericArg("Garbage collection percentage", 0, 100, 1000) // read data from file case "-input": if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: Input file name is missing\n") os.Exit(1) } fileName = args[1] // skip past first of two arguments args = args[1:] // data element for indexing case "-index": if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: Index element is missing\n") os.Exit(1) } indx = args[1] // skip past first of two arguments args = args[1:] // data element for indexing case "-local": if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: Data path is missing\n") os.Exit(1) } local = args[1] // skip past first of two arguments args = args[1:] // data cleanup flags case "-compress": doCompress = true case "-cleanup": doCleanup = true // debugging flags case "-debug": dbug = true case "-empty": mpty = true case "-ident": idnt = true case "-stats", "-stat": stts = true case "-timer": timr = true case "-profile": prfl = true case "-trial": trial = true case "-test": testCount = getNumericArg("Test data counter", 0, 0, 1000000) if len(args) > 1 { next := args[1] // if next argument is not another flag if next != "" && next[0] != '-' { // get optional data source specifier testType = next // skip past second of three arguments args = args[1:] } } default: // if not any of the controls, set flag to break out of for loop inSwitch = false } if !inSwitch { break } // skip past argument args = args[1:] if len(args) < 1 { break } } // reality checks on number of processors to use // performance degrades if capacity is above maximum number of partitions per second (context switching?) if numProcs == 0 { if defProcs > 0 { numProcs = defProcs } else { // best performance measurement with current code is obtained when 4 to 6 processors are assigned, // varying slightly among queries on PubmedArticle, gene DocumentSummary, and INSDSeq sequence records numProcs = 4 } } if numProcs > ncpu { numProcs = ncpu } if numProcs > maxProcs { numProcs = maxProcs } // allow simultaneous threads for multiplexed go routines runtime.GOMAXPROCS(numProcs) // adjust garbage collection target percentage if goGc >= 100 { debug.SetGCPercent(goGc) } // explicit -serv argument overrides -cons ratio if numServers > 0 { serverRatio = numServers / numProcs // if numServers / numProcs is not a whole number, do not print serverRatio in -stats if numServers != numProcs*serverRatio { serverRatio = 0 } } else { numServers = numProcs * serverRatio } // server limits if numServers > 128 { numServers = 128 } else if numServers < 1 { numServers = numProcs } // explicit -chan argument overrides default to number of servers if chanDepth == 0 { chanDepth = numServers } // -stats prints number of CPUs and performance tuning values if no other arguments (undocumented) if stts && len(args) < 1 { fmt.Fprintf(os.Stderr, "CPUs %d\n", ncpu) fmt.Fprintf(os.Stderr, "Proc %d\n", numProcs) if serverRatio > 0 { fmt.Fprintf(os.Stderr, "Cons %d\n", serverRatio) } fmt.Fprintf(os.Stderr, "Serv %d\n", numServers) fmt.Fprintf(os.Stderr, "Chan %d\n", chanDepth) fmt.Fprintf(os.Stderr, "Heap %d\n", heapSize) fmt.Fprintf(os.Stderr, "Farm %d\n", farmSize) if goGc >= 100 { fmt.Fprintf(os.Stderr, "Gogc %d\n", goGc) } fmt.Fprintf(os.Stderr, "\n") return } // -test N [pubmed|protein|insd|gene] repeats simple query on local XML to measure performance independent of stdin (undocumented) if testCount > 0 { var acc []string // select internal XML data source switch testType { case "pubmed": testString = pubMedArtSample case "protein", "sequence": testString = insdSeqSample case "insd": testString = insdSeqSample case "gene", "docsum": testString = geneDocSumSample default: testString = pubMedArtSample } // default commands if no other arguments if len(args) < 1 { switch testType { case "pubmed": acc = append(acc, "-pattern", "PubmedArticle", "-element", "LastName") case "protein", "sequence": acc = append(acc, "-pattern", "INSDSeq", "-element", "INSDSeq_accession-version") case "insd": acc = append(acc, "-insd", "mat_peptide", "%peptide", "product", "peptide") case "gene", "docsum": acc = append(acc, "-pattern", "DocumentSummary", "-element", "Name") default: acc = append(acc, "-pattern", "PubmedArticle", "-element", "LastName") } } // otherwise use remaining arguments for extraction commands for len(args) > 0 { acc = append(acc, args[0]) args = args[1:] } args = acc } if len(args) < 1 { fmt.Fprintf(os.Stderr, "\nERROR: Insufficient command-line arguments supplied to xtract\n") os.Exit(1) } // DOCUMENTATION COMMANDS inSwitch = true switch args[0] { case "-version": fmt.Printf("%s\n", xtractVersion) case "-help", "-extras", "-extra": fmt.Printf("xtract %s\n%s\n", xtractVersion, xtractHelp) case "-examples", "-example", "-scripts", "-script": fmt.Printf("xtract %s\n%s\n", xtractVersion, xtractExamples) case "-internal", "-internals": fmt.Printf("xtract %s\n%s\n", xtractVersion, xtractInternal) case "-sample", "-samples": // -sample [pubmed|protein|gene] sends specified sample record to stdout (undocumented) if len(args) > 1 { testType = args[1] } switch testType { case "pubmed": fmt.Printf("%s\n", pubMedArtSample) case "protein", "sequence", "insd": fmt.Printf("%s\n", insdSeqSample) case "gene", "docsum": fmt.Printf("%s\n", geneDocSumSample) default: fmt.Printf("%s\n", pubMedArtSample) } case "-keys": fmt.Printf("%s\n", keyboardShortcuts) case "-unix": fmt.Printf("%s\n", unixCommands) default: // if not any of the documentation commands, keep going inSwitch = false } if inSwitch { return } // INITIALIZE TABLES tbls := InitTables() if tbls == nil { fmt.Fprintf(os.Stderr, "\nERROR: Problem creating token streamer lookup tables\n") os.Exit(1) } // additional fields passed in master table tbls.ChanDepth = chanDepth tbls.FarmSize = farmSize // FILE NAME CAN BE SUPPLIED WITH -input COMMAND in := os.Stdin // check for data being piped into stdin fi, _ := os.Stdin.Stat() isPipe := bool((fi.Mode() & os.ModeNamedPipe) != 0) usingFile := false if fileName != "" { inFile, err := os.Open(fileName) if err != nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to open input file '%s'\n", fileName) os.Exit(1) } defer inFile.Close() // use indicated file instead of stdin in = inFile usingFile = true if isPipe && runtime.GOOS != "windows" { mode := fi.Mode().String() fmt.Fprintf(os.Stderr, "\nERROR: Input data from both stdin and file '%s', mode is '%s'\n", fileName, mode) os.Exit(1) } } // check for -input command after extraction arguments for _, str := range args { if str == "-input" { fmt.Fprintf(os.Stderr, "\nERROR: Misplaced -input command\n") os.Exit(1) } } // CREATE XML BLOCK READER FROM STDIN OR FILE rdr := NewXMLReader(in, doCompress, doCleanup) if rdr == nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to create XML Block Reader\n") os.Exit(1) } // SEQUENCE RECORD EXTRACTION COMMAND GENERATOR // -insd simplifies extraction of INSDSeq qualifiers if args[0] == "-insd" || args[0] == "-insd-" { addDash := true // -insd- variant suppresses use of dash as placeholder for missing qualifiers (undocumented) if args[0] == "-insd-" { addDash = false } args = args[1:] insd := ProcessINSD(args, isPipe || usingFile || testCount > 0, addDash) if !isPipe && !usingFile && testCount < 1 { // no piped input, so write output instructions fmt.Printf("xtract") for _, str := range insd { fmt.Printf(" %s", str) } fmt.Printf("\n") return } // data in pipe, so replace arguments, execute dynamically args = insd } // CITATION MATCHER EXTRACTION COMMAND GENERATOR // -hydra filters HydraResponse output by relevance score (undocumented) if args[0] == "-hydra" { hydra := ProcessHydra(isPipe || usingFile) if !isPipe && !usingFile { // no piped input, so write output instructions fmt.Printf("xtract") for _, str := range hydra { fmt.Printf(" %s", str) } fmt.Printf("\n") return } // data in pipe, so replace arguments, execute dynamically args = hydra } // XML DATA FORMATTING/COMPRESSION COMMAND GENERATOR // -reformat takes a parent pattern and compresses each object for fastest processing (undocumented) if args[0] == "-reformat" { args = args[1:] max := len(args) if max < 1 { fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -reformat command\n") os.Exit(1) } // required first argument is parent pattern, will explore using Parent/* construct, write component with -element "*" prnt := args[0] if prnt == "" { fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -reformat command\n") os.Exit(1) } if prnt == "-xml" || prnt == "-doctype" || prnt == "-pfx" || prnt == "-sfx" { fmt.Fprintf(os.Stderr, "\nERROR: Deprecated argument '%s' used in -reformat command\n", prnt) os.Exit(1) } // optional second argument controls XML expansion or compression level // asterisks MUST be quoted to avoid interpration as file wildcard by Unix shell elm := "*" addRet := false hideDoctype := false hideWrapper := false if max > 1 { // * = compact, ** = flush, *** = indented, **** = subtree, ***** = attributes on separate lines // @ = remove attributes // ^ = suppress xml and doctype, ^^ = also suppress parent set wrapper numStars := 0 numCarets := 0 hideAttrs := false for _, ch := range args[1] { if ch == '*' { numStars++ } else if ch == '@' { hideAttrs = true } else if ch == '^' { numCarets++ } } if numStars > 1 { addRet = true } if numCarets > 0 { hideDoctype = true if numCarets > 1 { hideWrapper = true } } // construct legal element argument for PrintSubtree switch numStars { case 1: elm = "*" case 2: elm = "**" case 3: elm = "***" case 4: elm = "****" case 5: elm = "*****" default: elm = "*" } if hideAttrs { elm += "@" } } // optional third argument provides detailed DOCTYPE construct doctype := "" if max > 2 { str := ConvertSlash(args[2]) if strings.HasPrefix(str, "") { doctype = str } } if !isPipe && !usingFile { // no piped input, so write output instructions (without -head and -tail arguments) if addRet { fmt.Printf("xtract -pattern %s/* -ret \"\" -element \"%s\"\n", prnt, elm) } else { fmt.Printf("xtract -pattern %s/* -element \"%s\"\n", prnt, elm) } return } // add xml, DOCTYPE, and lines at the beginning hd := fmt.Sprintf("\n\n<%s>", prnt, prnt) if doctype != "" { // use supplied DOCTYPE argument hd = fmt.Sprintf("\n%s\n<%s>", doctype, prnt) } if hideDoctype { // or just line hd = fmt.Sprintf("<%s>", prnt) } // add line at the end tl := fmt.Sprintf("", prnt) // use -pattern Parent/* construct prnt += "/*" var acc []string if !hideWrapper { acc = append(acc, "-head", hd, "-tail", tl) } acc = append(acc, "-pattern", prnt) if addRet { acc = append(acc, "-ret", "") } acc = append(acc, "-element", elm) // data in pipe, so replace arguments, execute dynamically args = acc } // CONFIRM INPUT DATA AVAILABILITY AFTER RUNNING COMMAND GENERATORS if fileName == "" && runtime.GOOS != "windows" { fromStdin := bool((fi.Mode() & os.ModeCharDevice) == 0) if !isPipe || !fromStdin { mode := fi.Mode().String() fmt.Fprintf(os.Stderr, "\nERROR: No data supplied to xtract from stdin or file, mode is '%s'\n", mode) os.Exit(1) } } if testCount < 1 && !usingFile && !isPipe { fmt.Fprintf(os.Stderr, "\nERROR: No XML input data supplied to xtract\n") os.Exit(1) } // START PROFILING IF REQUESTED if prfl { dbug = true f, err := os.Create("cpu.pprof") if err != nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to create profile output file\n") os.Exit(1) } pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() } // INITIALIZE PROCESS TIMER AND RECORD COUNT startTime := time.Now() recordCount := 0 byteCount := 0 // function to print processing rate and program duration printDuration := func(name string) { stopTime := time.Now() duration := stopTime.Sub(startTime) seconds := float64(duration.Nanoseconds()) / 1e9 if recordCount >= 1000000 { fmt.Fprintf(os.Stderr, "\nXtract processed %d million %s in %.3f seconds", recordCount/1000000, name, seconds) } else { fmt.Fprintf(os.Stderr, "\nXtract processed %d %s in %.3f seconds", recordCount, name, seconds) } if seconds >= 0.001 && recordCount > 0 { rate := int(float64(recordCount) / seconds) if rate >= 1000000 { fmt.Fprintf(os.Stderr, " (%d mega%s/second", rate/1000000, name) } else { fmt.Fprintf(os.Stderr, " (%d %s/second", rate, name) } if byteCount > 0 { rate := int(float64(byteCount) / seconds) if rate >= 1000000 { fmt.Fprintf(os.Stderr, ", %d megabytes/second", rate/1000000) } else if rate >= 1000 { fmt.Fprintf(os.Stderr, ", %d kilobytes/second", rate/1000) } else { fmt.Fprintf(os.Stderr, ", %d bytes/second", rate) } } fmt.Fprintf(os.Stderr, ")") } fmt.Fprintf(os.Stderr, "\n\n") } // PERFORMANCE TIMING COMMANDS inSwitch = true action := NOPROCESS recordType := "" switch args[0] { case "-chunk": action = DOCHUNK recordType = "blocks" case "-split": action = DOSPLIT recordType = "patterns" case "-drain": action = DODRAIN recordType = "patterns" case "-token": action = DOTOKEN recordType = "tokens" default: // if not any of the formatting commands, keep going inSwitch = false } if inSwitch { recordCount, byteCount = ProcessXMLStream(rdr, tbls, args, action) printDuration(recordType) return } // SPECIAL FORMATTING COMMANDS inSwitch = true action = NOPROCESS switch args[0] { case "-format": action = DOFORMAT case "-outline": action = DOOUTLINE case "-synopsis": action = DOSYNOPSIS case "-verify", "-validate": action = DOVERIFY case "-filter": action = DOFILTER default: // if not any of the formatting commands, keep going inSwitch = false } if inSwitch { ProcessXMLStream(rdr, tbls, args, action) return } // SPECIFY STRINGS TO GO BEFORE AND AFTER ENTIRE OUTPUT OR EACH RECORD head := "" tail := "" hd := "" tl := "" for { inSwitch = true switch args[0] { case "-head": if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -head command\n") os.Exit(1) } head = ConvertSlash(args[1]) case "-tail": if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -tail command\n") os.Exit(1) } tail = ConvertSlash(args[1]) case "-hd": if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -hd command\n") os.Exit(1) } hd = ConvertSlash(args[1]) case "-tl": if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: Pattern missing after -tl command\n") os.Exit(1) } tl = ConvertSlash(args[1]) default: // if not any of the controls, set flag to break out of for loop inSwitch = false } if !inSwitch { break } // skip past arguments args = args[2:] if len(args) < 1 { fmt.Fprintf(os.Stderr, "\nERROR: Insufficient command-line arguments supplied to xtract\n") os.Exit(1) } } // per-record head and tail passed in master table tbls.Hd = hd tbls.Tl = tl // ENSURE PRESENCE OF PATTERN ARGUMENT if len(args) < 1 { fmt.Fprintf(os.Stderr, "\nERROR: Insufficient command-line arguments supplied to xtract\n") os.Exit(1) } // make sure top-level -pattern command is next if args[0] != "-pattern" && args[0] != "-Pattern" { fmt.Fprintf(os.Stderr, "\nERROR: No -pattern in command-line arguments\n") os.Exit(1) } if len(args) < 2 { fmt.Fprintf(os.Stderr, "\nERROR: Item missing after -pattern command\n") os.Exit(1) } topPat := args[1] if topPat == "" { fmt.Fprintf(os.Stderr, "\nERROR: Item missing after -pattern command\n") os.Exit(1) } if strings.HasPrefix(topPat, "-") { fmt.Fprintf(os.Stderr, "\nERROR: Misplaced %s command\n", topPat) os.Exit(1) } // look for -pattern Parent/* construct for heterogeneous data, e.g., -pattern PubmedArticleSet/* topPattern, star := SplitInTwoAt(topPat, "/", LEFT) if topPattern == "" { return } parent := "" if star == "*" { parent = topPattern } else if star != "" { fmt.Fprintf(os.Stderr, "\nERROR: -pattern Parent/Child construct is not supported\n") os.Exit(1) } // SAVE XML COMPONENT RECORDS TO LOCAL DIRECTORY INDEXED BY TRIE ON IDENTIFIER // -local plus -index saves XML files in trie-based directory structure (experimental) if local != "" && indx != "" { prnt, match := SplitInTwoAt(indx, "/", RIGHT) match, attrib := SplitInTwoAt(match, "@", LEFT) var idbuf bytes.Buffer makeTrie := func(str string) string { idbuf.Reset() between := "" for _, ch := range str { if ch == '.' { break } idbuf.WriteString(between) idbuf.WriteRune(ch) between = "/" } trie := idbuf.String() return trie } PartitionPattern(topPattern, star, rdr, func(rec int, ofs int64, str string) { pat, ok := ParseXML(str[:], parent, tbls) if !ok { return } trie := "" file := "" num := 0 ExploreElements(pat, "", prnt, match, attrib, false, 1, func(stn string, lvl int) { num++ if stn != "" { trie = makeTrie(stn) file = stn } }) if trie == "" || file == "" || num != 1 { return } dpath := path.Join(local, trie) if dpath == "" { return } _, err := os.Stat(dpath) if err != nil && os.IsNotExist(err) { err = os.MkdirAll(dpath, os.ModePerm) } if err != nil { fmt.Println(err.Error()) return } fpath := path.Join(dpath, file+".xml") if fpath == "" { return } _, err = os.Stat(fpath) if err != nil && os.IsNotExist(err) { fl, err := os.Create(fpath) if err != nil { fmt.Println(err.Error()) return } fl.Close() } fl, err := os.OpenFile(fpath, os.O_RDWR, 0644) if err != nil { fmt.Println(err.Error()) return } fl.WriteString(str) fl.WriteString("\n") err = fl.Sync() if err != nil { fmt.Println(err.Error()) } fl.Close() fmt.Printf("%d\t%s\n", rec, fpath) }) return } // GENERATE RECORD INDEX ON XML INPUT FILE // -index command prints record identifier, file offset, and XML size if indx != "" { prnt, match := SplitInTwoAt(indx, "/", RIGHT) match, attrib := SplitInTwoAt(match, "@", LEFT) // legend := "REC\tID\tOFST\tSIZE" PartitionPattern(topPattern, star, rdr, func(rec int, ofs int64, str string) { pat, ok := ParseXML(str[:], parent, tbls) if !ok { return } id := "" num := 0 ExploreElements(pat, "", prnt, match, attrib, false, 1, func(stn string, lvl int) { num++ if stn != "" { id = stn } }) if id == "" || num != 1 { return } /* if legend != "" { fmt.Printf("%s\n", legend) legend = "" } */ fmt.Printf("%d\t%s\t%d\t%d\n", rec, id, ofs, len(str)) }) return } // PARSE AND VALIDATE EXTRACTION ARGUMENTS // parse nested exploration instruction from command-line arguments cmds := ParseArguments(args, topPattern) if cmds == nil { fmt.Fprintf(os.Stderr, "\nERROR: Problem parsing command-line arguments\n") os.Exit(1) } // PERFORMANCE TIMING COMMAND // -stats with an extraction command prints XML size and processing time for each record if stts { legend := "REC\tOFST\tSIZE\tTIME" PartitionPattern(topPattern, star, rdr, func(rec int, ofs int64, str string) { beginTime := time.Now() ProcessQuery(str[:], parent, rec, cmds, tbls) endTime := time.Now() duration := endTime.Sub(beginTime) micro := int(float64(duration.Nanoseconds()) / 1e3) if legend != "" { fmt.Printf("%s\n", legend) legend = "" } fmt.Printf("%d\t%d\t%d\t%d\n", rec, ofs, len(str), micro) }) return } // PERFORMANCE OPTIMIZATION FUNCTIONS // -test N runs a test extraction N times for each -proc from 1 to nCPU (undocumented) if testCount > 0 && testString != "" { // clean up copy of sample string included in source code sample := strings.TrimSpace(testString) sample = CleanupBadSpaces(sample) legend := "CPU\tTIME\tRATE" for numServ := 1; numServ <= ncpu; numServ++ { runtime.GOMAXPROCS(numServ) // alternative producer sends sample XML through channel N times xmlq := make(chan Extract, chanDepth) go func(out chan<- Extract) { for rec := 1; rec <= testCount; rec++ { out <- Extract{rec, sample} } close(out) }(xmlq) tblq := CreateConsumers(cmds, tbls, parent, numServ, xmlq) if xmlq == nil || tblq == nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to create servers\n") os.Exit(1) } begTime := time.Now() recordCount = 0 for _ = range tblq { recordCount++ runtime.Gosched() } debug.FreeOSMemory() endTime := time.Now() expended := endTime.Sub(begTime) secs := float64(expended.Nanoseconds()) / 1e9 if secs >= 0.000001 && recordCount > 0 { speed := int(float64(recordCount) / secs) if legend != "" { fmt.Printf("%s\n", legend) legend = "" } fmt.Printf("%d\t%.3f\t%d\n", numServ, secs, speed) } } return } // -trial -input fileName runs the specified extraction for each -proc from 1 to nCPU if trial && fileName != "" { legend := "CPU\tRATE\tDEV" for numServ := 1; numServ <= ncpu; numServ++ { runtime.GOMAXPROCS(numServ) sum := 0 count := 0 mean := 0.0 m2 := 0.0 // calculate mean and standard deviation of processing rate for trials := 0; trials < 5; trials++ { inFile, err := os.Open(fileName) if err != nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to open input file '%s'\n", fileName) os.Exit(1) } rdr := NewXMLReader(inFile, doCompress, doCleanup) if rdr == nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to read input file\n") os.Exit(1) } xmlq := CreateProducer(topPattern, star, rdr, tbls) tblq := CreateConsumers(cmds, tbls, parent, numServ, xmlq) if xmlq == nil || tblq == nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to create servers\n") os.Exit(1) } begTime := time.Now() recordCount = 0 for _ = range tblq { recordCount++ runtime.Gosched() } inFile.Close() debug.FreeOSMemory() endTime := time.Now() expended := endTime.Sub(begTime) secs := float64(expended.Nanoseconds()) / 1e9 if secs >= 0.000001 && recordCount > 0 { speed := int(float64(recordCount) / secs) sum += speed count++ x := float64(speed) delta := x - mean mean += delta / float64(count) m2 += delta * (x - mean) } } if legend != "" { fmt.Printf("%s\n", legend) legend = "" } if count > 1 { vrc := m2 / float64(count-1) dev := int(math.Sqrt(vrc)) fmt.Printf("%d\t%d\t%d\n", numServ, sum/count, dev) } } return } // PROCESS SINGLE SELECTED RECORD IF -pattern ARGUMENT IS IMMEDIATELY FOLLOWED BY -position COMMAND if cmds.Visit == topPat && cmds.Position != "" { qry := "" idx := 0 if cmds.Position == "first" { PartitionPattern(topPattern, star, rdr, func(rec int, ofs int64, str string) { if rec == 1 { qry = str idx = rec } }) } else if cmds.Position == "last" { PartitionPattern(topPattern, star, rdr, func(rec int, ofs int64, str string) { qry = str idx = rec }) } else { // use numeric position number, err := strconv.Atoi(cmds.Position) if err != nil { fmt.Fprintf(os.Stderr, "\nERROR: Unrecognized position '%s'\n", cmds.Position) os.Exit(1) } PartitionPattern(topPattern, star, rdr, func(rec int, ofs int64, str string) { if rec == number { qry = str idx = rec } }) } if qry == "" { return } // clear position on top node to prevent condition test failure cmds.Position = "" // process single selected record res := ProcessQuery(qry[:], parent, idx, cmds, tbls) if res != "" { fmt.Printf("%s\n", res) } return } // LAUNCH PRODUCER AND CONSUMER SERVERS // launch producer goroutine to partition XML by pattern xmlq := CreateProducer(topPattern, star, rdr, tbls) if xmlq == nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to create producer\n") os.Exit(1) } // launch consumer goroutines to parse and explore partitioned XML objects tblq := CreateConsumers(cmds, tbls, parent, numServers, xmlq) if tblq == nil { fmt.Fprintf(os.Stderr, "\nERROR: Unable to create consumers\n") os.Exit(1) } // PERFORMANCE SUMMARY if dbug { // drain results, but suppress extraction output for ext := range tblq { byteCount += len(ext.Text) recordCount++ runtime.Gosched() } // force garbage collection, return memory to operating system debug.FreeOSMemory() // print processing parameters as XML object stopTime := time.Now() duration := stopTime.Sub(startTime) seconds := float64(duration.Nanoseconds()) / 1e9 // Threads is a more easily explained concept than GOMAXPROCS fmt.Printf("\n") fmt.Printf(" %d\n", numProcs) fmt.Printf(" %d\n", numServers) fmt.Printf(" \n", seconds) if seconds >= 0.001 && recordCount > 0 { rate := int(float64(recordCount) / seconds) fmt.Printf(" %d\n", rate) } fmt.Printf("\n") return } // DRAIN OUTPUT CHANNEL TO EXECUTE EXTRACTION COMMANDS, RESTORE OUTPUT ORDER WITH HEAP // initialize empty heap hp := &ExtractHeap{} heap.Init(hp) // index of next desired result next := 1 delay := 0 var buffer bytes.Buffer count := 0 okay := false if head != "" { buffer.WriteString(head[:]) buffer.WriteString("\n") } // printResult prints output for current pattern, handles -empty and -ident flags, and periodically flushes buffer printResult := func(curr Extract) { str := curr.Text if mpty { if str == "" { okay = true idx := curr.Index val := strconv.Itoa(idx) buffer.WriteString(val[:]) buffer.WriteString("\n") count++ } } else if str != "" { okay = true if idnt { idx := curr.Index val := strconv.Itoa(idx) buffer.WriteString(val[:]) buffer.WriteString("\t") } // save output to byte buffer buffer.WriteString(str[:]) count++ } if count > 1000 { count = 0 txt := buffer.String() if txt != "" { // print current buffer os.Stdout.WriteString(txt[:]) } buffer.Reset() } } for ext := range tblq { // push result onto heap heap.Push(hp, ext) // read several values before checking to see if next record to print has been processed if delay < heapSize { delay++ continue } delay = 0 for hp.Len() > 0 { // remove lowest item from heap, use interface type assertion curr := heap.Pop(hp).(Extract) if curr.Index == next { // if this is the desired item, send to output printResult(curr) recordCount++ // increment index next++ // and keep checking heap to see if next result is already available } else { // otherwise push back onto heap heap.Push(hp, curr) // and go back to waiting on input channel break } } } // send remainder of heap to output for hp.Len() > 0 { curr := heap.Pop(hp).(Extract) printResult(curr) recordCount++ } if tail != "" { buffer.WriteString(tail[:]) buffer.WriteString("\n") } // do not print head or tail if no extraction output if okay { txt := buffer.String() if txt != "" { // print final buffer os.Stdout.WriteString(txt[:]) } } buffer.Reset() // force garbage collection and return memory before calculating processing rate debug.FreeOSMemory() if timr { printDuration("records") } } edirect/word-at-a-time0000755000725300123250000000012012551320074015151 0ustar coremakecppcore#!/bin/bash -norc sed 's/[^a-zA-Z0-9]/ /g; s/^ *//' | tr 'A-Z' 'a-z' | fmt -w 1 edirect/nquire0000755000725300123250000002373713027672134013756 0ustar coremakecppcore#!/usr/bin/perl # =========================================================================== # # PUBLIC DOMAIN NOTICE # National Center for Biotechnology Information (NCBI) # # This software/database is a "United States Government Work" under the # terms of the United States Copyright Act. It was written as part of # the author's official duties as a United States Government employee and # thus cannot be copyrighted. This software/database is freely available # to the public for use. The National Library of Medicine and the U.S. # Government do not place any restriction on its use or reproduction. # We would, however, appreciate having the NCBI and the author cited in # any work or product based on this material. # # Although all reasonable efforts have been taken to ensure the accuracy # and reliability of the software and data, the NLM and the U.S. # Government do not and cannot warrant the performance or results that # may be obtained by using this software or data. The NLM and the U.S. # Government disclaim all warranties, express or implied, including # warranties of performance, merchantability or fitness for any particular # purpose. # # =========================================================================== # # File Name: nquire # # Author: Jonathan Kans # # Version Creation Date: 8/20/12 # # ========================================================================== # Entrez Direct - EDirect # use strict; use warnings; my ($LibDir, $ScriptName); use File::Spec; BEGIN { my $Volume; ($Volume, $LibDir, $ScriptName) = File::Spec->splitpath($0); $LibDir = File::Spec->catpath($Volume, $LibDir, ''); if (my $RealPathname = eval {readlink $0}) { do { $RealPathname = File::Spec->rel2abs($RealPathname, $LibDir); ($Volume, $LibDir, undef) = File::Spec->splitpath($RealPathname); $LibDir = File::Spec->catpath($Volume, $LibDir, '') } while ($RealPathname = eval {readlink $RealPathname}); } else { $LibDir = File::Spec->rel2abs($LibDir) } $LibDir .= '/aux/lib/perl5'; } use lib $LibDir; use LWP::UserAgent; use POSIX; use URI::Escape; # definitions use constant false => 0; use constant true => 1; # utility subroutines sub clearflags { %macros = (); $agent = "Nquire/1.0"; $alias = ""; $debug = false; $http = ""; $output = ""; } sub map_macros { $qury = shift (@_); if ( $qury !~ /\(#/ ) { return $qury; } if ( scalar (keys %macros) > 0 ) { for ( keys %macros ) { $ky = $_; $vl = $macros{$_}; $qury =~ s/\((\#$ky)\)/$vl/g; } } return $qury; } sub read_aliases { if ( $alias ne "" ) { if (open (my $PROXY_IN, $alias)) { while ( $thisline = <$PROXY_IN> ) { $thisline =~ s/\r//; $thisline =~ s/\n//; $thisline =~ s/ +/ /g; $thisline =~ s/> 300); $usragnt->agent( "$agent" ); $res = $usragnt->get ( $urlx ); if ( $res->is_success) { $rslt = $res->content; } else { print STDERR $res->status_line . "\n"; } if ( $rslt eq "" ) { print STDERR "No do_get output returned from '$urlx'\n"; } if ( $debug ) { print STDERR "$rslt\n"; } return $rslt; } $usragnt = new LWP::UserAgent (timeout => 300); $usragnt->agent( "$agent" ); $req = new HTTP::Request POST => "$urlx"; $req->content_type('application/x-www-form-urlencoded'); $req->content("$argx"); $res = $usragnt->request ( $req ); if ( $res->is_success) { $rslt = $res->content; } else { print STDERR $res->status_line . "\n"; } if ( $rslt eq "" ) { if ( $argx ne "" ) { $urlx .= "?"; $urlx .= "$argx"; } print STDERR "No do_post output returned from '$urlx'\n"; } if ( $debug ) { print STDERR "$rslt\n"; } return $rslt; } # uri_escape with backslash exceptions sub do_uri_escape { $patx = shift (@_); $rslt = ""; while ( $patx ne "" ) { if ( $patx =~ /^\\\\(.+)/ ) { $rslt .= "\\"; $patx = $1; } elsif ( $patx =~ /^\\(.)(.+)/ ) { $rslt .= $1; $patx = $2; } elsif ( $patx =~ /^(.)(.+)/ ) { $rslt .= uri_escape ($1); $patx = $2; } elsif ( $patx =~ /^(.)/ ) { $rslt .= uri_escape ($1); $patx = ""; } } return $rslt; } # nquire executes an external URL query from command line arguments my $nquire_help = qq{ Query Commands -get Uses HTTP GET instead of POST -url Base URL for external search Examples nquire -get -url "http://collections.mnh.si.edu/services/resolver/resolver.php" \\ -voucher "Birds:625456" | xtract -pattern Result -element ScientificName Country nquire -get -url http://w1.weather.gov/xml/current_obs/KSFO.xml | xtract -pattern current_observation -tab "\\n" \\ -element weather temp_f wind_dir wind_mph nquire -eutils efetch.fcgi -db pubmed -id 2539356 -rettype medline -retmode text nquire -eutils esummary.fcgi -db pubmed -id 2539356 -version 2.0 nquire -url "https://eutils.ncbi.nlm.nih.gov/entrez/eutils" elink.fcgi \\ -dbfrom protein -db protein -cmd neighbor -linkname protein_protein -id NP_476532.1 }; sub nquire { # nquire -url http://... -tag value -tag value | ... $url = ""; $arg = ""; $pfx = ""; $amp = ""; $pat = ""; @args = @ARGV; $max = scalar @args; if ( $max > 0 and $ARGV[0] eq "-help" ) { print $nquire_help; return; } if ( $max < 2 ) { return; } $i = 0; # if present, -debug must be first argument, only prints generated URL (undocumented) if ( $i < $max ) { $pat = $args[$i]; if ( $pat eq "-debug" ) { $i++; $debug = true; } } # if present, -http get or -get must be next # nquire -get -url "http://collections.mnh.si.edu/services/resolver/resolver.php" -voucher "Birds:625456" if ( $i < $max ) { $pat = $args[$i]; if ( $pat eq "-http" ) { $i++; if ( $i < $max ) { $http = $args[$i]; $i++; } } elsif ( $pat eq "-get" ) { $i++; $http = "get"; } } # if present, -agent must be next argument (undocumented) if ( $i < $max ) { $pat = $args[$i]; if ( $pat eq "-agent" ) { $i++; if ( $i < $max ) { $agent = $args[$i]; $i++; } } } # read file of keyword shortcuts for URL expansion if ( $i < $max ) { $pat = $args[$i]; if ( $pat eq "-alias" ) { $i++; if ( $i < $max ) { $alias = $args[$i]; if ( $alias ne "" ) { read_aliases (); } $i++; } } } # read URL if ( $i < $max ) { $pat = $args[$i]; if ( $pat eq "-url" ) { $i++; if ( $i < $max ) { $url = $args[$i]; $url = map_macros ($url); $i++; } } elsif ( $pat eq "-ncbi" ) { # shortcut for ncbi base (undocumented) $i++; if ( $i < $max ) { $url = "https://www.ncbi.nlm.nih.gov"; } } elsif ( $pat eq "-eutils" ) { # shortcut for eutils base (undocumented) $i++; if ( $i < $max ) { $url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"; } } elsif ( $pat eq "-test" ) { # shortcut for eutilstest base (undocumented) $i++; if ( $i < $max ) { $url = "https://eutilstest.ncbi.nlm.nih.gov/entrez/eutils"; } } elsif ( $pat eq "-hydra" ) { # internal citation match request (undocumented) $i++; if ( $i < $max ) { $url = "https://www.ncbi.nlm.nih.gov/projects/hydra/hydra_search.cgi"; $pat = $args[$i]; $pat = map_macros ($pat); $enc = do_uri_escape ($pat); $arg="search=pubmed_search_citation_top_20.1&query=$enc"; $amp = "&"; $i++; } } elsif ( $pat eq "-revhist" ) { # internal sequence revision history request (undocumented) $i++; if ( $i < $max ) { $url = "https://www.ncbi.nlm.nih.gov/sviewer/girevhist.cgi"; $pat = $args[$i]; $arg="cmd=seqid&txt=on&seqid=asntext&os=PUBSEQ_OS&val=$pat"; $amp = "&"; $i++; } } } if ( $url eq "" ) { return; } # hard-coded URL aliases for common NCBI web sites if ( $url =~ /\(#/ ) { $ky = "ncbi_url"; if ( $url =~ /\(#$ky\)/ ) { $vl = "https://www.ncbi.nlm.nih.gov"; $url =~ s/\((\#$ky)\)/$vl/g; } $ky = "eutils_url"; if ( $url =~ /\(#$ky\)/ ) { $vl = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"; $url =~ s/\((\#$ky)\)/$vl/g; } } # arguments before next minus are added to base URL as /value $go_on = true; while ( $i < $max and $go_on ) { $pat = $args[$i]; if ( $pat =~ /^-(.+)/ ) { $go_on = false; } else { $pat = map_macros ($pat); $url .= "/" . $pat; $i++; } } # now expect tag with minus and value[s] without, add as &tag=value[,value] while ( $i < $max ) { $pat = $args[$i]; if ( $pat =~ /^-(.+)/ ) { $pat = $1; $pfx = $amp . "$pat="; $amp = ""; } else { $pat =~ s/^\\-/-/g; $pat = map_macros ($pat); $enc = do_uri_escape ($pat); $arg .= $pfx . $enc; $pfx = ","; $amp = "&"; } $i++; } if ( $debug ) { if ( $arg eq "" ) { print "$url\n"; } else { print "$url?$arg\n"; } return; } $output = do_post ($url, $arg); print "$output"; } # initialize clearflags (); # execute URL request nquire (); # close input and output files close (STDIN); close (STDOUT); close (STDERR); edirect/econtact0000755000725300123250000000041112122445476014235 0ustar coremakecppcore#!/bin/sh dir=`dirname "$0"` case "`uname -s`" in CYGWIN_NT*) # Use a negative match here because the shell treats 0 as success. if perl -e 'exit $^O !~ /^MSWin/'; then dir=`cygpath -w "$dir"` fi ;; esac exec perl "$dir"/edirect.pl -contact "$@" edirect/xy-plot0000755000725300123250000000236713006466653014067 0ustar coremakecppcore#!/bin/sh # For Mac, please obtain command-line-enabled Plot2x from http://apps.micw.org/apps/plot2/downloads.php # For Unix or PC/Cygwin, please obtain gnuplot from http://gnuplot.sourceforge.net/download.html plot2x= p2x_path=/Applications/Plot2x.app/Contents/MacOS/Plot2x for pfx in "$HOME" '' do if [ -x "$pfx$p2x_path" ] then plot2x=$pfx$p2x_path break fi done if [ -n "$plot2x" ] then cat > "$HOME/Desktop/edirect.dat" outfile="$HOME/Desktop/edirect.png" if [ -n "$*" ] then outfile="$*" fi cat > "$HOME/Desktop/edirect.macro" </dev/null then cat > "edirect.dat" outfile="edirect.png" if [ -n "$*" ] then outfile="$*" fi gnuplot -e "set terminal png; set output '$outfile'; unset key; plot 'edirect.dat' with lines" rm "edirect.dat" else echo "To generate .png output please install either gnuplot or Plot2x" >&2 fi if [ -f "$outfile" ] then case "`uname -s`" in Darwin ) open "$outfile" ;; CYGWIN_NT* ) cygstart "$outfile" ;; * ) xdg-open "$outfile" 2>/dev/null ;; esac fi edirect/ftp-ls0000755000725300123250000000111413013652635013640 0ustar coremakecppcore#!/usr/bin/perl -w # Usage: ftp-ls SERVER PATH use strict; use Net::FTP; my $server = shift; my $dir = shift; my $ftp = new Net::FTP($server, Passive => 1) or die "Unable to connect to FTP server: $!"; $ftp->login or die "Unable to log in to FTP server"; $ftp->cwd($dir) or die "Unable to change to $dir"; my $contents = $ftp->dir; die "Unable to list contents" unless defined $contents; for (@$contents) { if (/^-.*?(\S*)$/) { print "$1\n"; } elsif (/^d.*?(\S*)$/) { print "$1/\n"; } elsif (/^l.*?(\S*) -> \S*$/) { print "$1@\n"; } } edirect/edirutil0000755000725300123250000000520012133560402014244 0ustar coremakecppcore#!/usr/bin/perl # =========================================================================== # # PUBLIC DOMAIN NOTICE # National Center for Biotechnology Information (NCBI) # # This software/database is a "United States Government Work" under the # terms of the United States Copyright Act. It was written as part of # the author's official duties as a United States Government employee and # thus cannot be copyrighted. This software/database is freely available # to the public for use. The National Library of Medicine and the U.S. # Government do not place any restriction on its use or reproduction. # We would, however, appreciate having the NCBI and the author cited in # any work or product based on this material. # # Although all reasonable efforts have been taken to ensure the accuracy # and reliability of the software and data, the NLM and the U.S. # Government do not and cannot warrant the performance or results that # may be obtained by using this software or data. The NLM and the U.S. # Government disclaim all warranties, express or implied, including # warranties of performance, merchantability or fitness for any particular # purpose. # # =========================================================================== # # File Name: edirutil # # Author: Jonathan Kans # # Version Creation Date: 8/20/12 # # ========================================================================== # Entrez Direct - EDirect # use strict; use warnings; use Getopt::Long; # definitions use constant false => 0; use constant true => 1; sub edirutil { my $db = ""; my $web = ""; my $key = 0; my $num = 0; my $stp = 0; my $version = ""; my $seconds = 0; my $silent = false; my $verbose = false; my $debug = false; GetOptions ( "db=s" => \$db, "web=s" => \$web, "key=i" => \$key, "count=i" => \$num, "step=i" => \$stp, "seconds=s" => \$seconds, "version=s" => \$version, "silent" => \$silent, "verbose" => \$verbose, "debug" => \$debug ); print "\n"; if ( $db ne "" ) { print " $db\n"; } if ( $web ne "" ) { print " $web\n"; } if ( $key ne "" ) { print " $key\n"; } if ( $num ne "" ) { print " $num\n"; } if ( $stp ne "" ) { print " $stp\n"; } if ( $silent ) { print " Y\n"; } if ( $verbose ) { print " Y\n"; } if ( $debug ) { print " Y\n"; } print "\n"; } edirutil (); close (STDIN); close (STDOUT); close (STDERR); edirect/xtract0000755000725300123250000000026413002461553013740 0ustar coremakecppcore#!/bin/sh PATH=/bin:/usr/bin export PATH compiled=$0.`uname -s | sed -e 's/[0-9.-]*$//'` if [ -x "$compiled" ] then exec "$compiled" "$@" else exec $0.pl -fallback "$@" fi edirect/enotify0000755000725300123250000000041012122445476014111 0ustar coremakecppcore#!/bin/sh dir=`dirname "$0"` case "`uname -s`" in CYGWIN_NT*) # Use a negative match here because the shell treats 0 as success. if perl -e 'exit $^O !~ /^MSWin/'; then dir=`cygpath -w "$dir"` fi ;; esac exec perl "$dir"/edirect.pl -notify "$@" edirect/ftp-cp0000755000725300123250000000067213013652635013634 0ustar coremakecppcore#!/usr/bin/perl -w # Usage: ftp-cp SERVER PATH FILE... use strict; use Net::FTP; my $server = shift; my $dir = shift; my $ftp = new Net::FTP($server, Passive => 1) or die "Unable to connect to FTP server: $!"; $ftp->login or die "Unable to log in to FTP server"; $ftp->cwd($dir) or die "Unable to change to $dir"; $ftp->binary or warn "Unable to set binary mode"; for (@ARGV) { $ftp->get($_) or die "Unable to retrieve $_"; } edirect/esummary0000755000725300123250000000042612276326030014277 0ustar coremakecppcore#!/bin/sh dir=`dirname "$0"` case "`uname -s`" in CYGWIN_NT*) # Use a negative match here because the shell treats 0 as success. if perl -e 'exit $^O !~ /^MSWin/'; then dir=`cygpath -w "$dir"` fi ;; esac exec perl "$dir"/edirect.pl -fetch -format docsum "$@" edirect/sort-uniq-count-rank0000755000725300123250000000047413017666764016477 0ustar coremakecppcore#!/bin/sh TAB=' ' # The text between the quotes should contain a single tab character. flags="f" if [ -n "$*" ] then flags=`echo " $*" | sed 's/[^bfinrs]//g'` if [ -z "$flags" ] then flags="s" fi fi sort "-$flags" | uniq -i -c | perl -pe 's/\s*(\d+)\s(.+)/$1\t$2/' | sort -t "$TAB" -k 1,1nr "-k 2$flags" edirect/between-two-genes0000755000725300123250000000013013003505115015754 0ustar coremakecppcore#!/bin/bash -norc awk -F '\t' -v 'OFS=\t' "/^$1\t/{a++}/^$2\t/{a++}a>0{print}a>1{exit}" edirect/README0000644000725300123250000000471513002766451013377 0ustar coremakecppcoreENTREZ DIRECT - README Entrez Direct (EDirect) is an advanced method for accessing the NCBI's set of interconnected Entrez databases (publication, nucleotide, protein, structure, gene, variation, expression, etc.) from a terminal window. It uses command-line arguments for the query terms and combines individual operations with UNIX pipes. EDirect also provides an argument-driven function that simplifies the extraction of data from document summaries or other results that are returned in XML format. Queries can move seamlessly between EDirect commands and UNIX utilities or scripts to perform actions that cannot be accomplished entirely within Entrez. EDirect consists of a set of scripts that are downloaded to the user's computer. If you extract the archive in your home directory, you may need to enter: PATH=$PATH:$HOME/edirect in a terminal window to temporarily add EDirect functions to the PATH environment variable so they can be run by name. You can then try EDirect by copying the sample query below and pasting it into the terminal window for execution: esearch -db pubmed -query "Beadle AND Tatum AND Neurospora" | elink -related | efilter -query "NOT historical article [FILT]" | efetch -format docsum | xtract -pattern DocumentSummary -if Author -and Title \ -element Id -first "Author/Name" -element Title | grep -i -e enzyme -e synthesis | sort -t $'\t' -k 2,3f | column -s $'\t' -t | head -n 10 | cut -c 1-80 This query returns the PubMed ID, first author name, and article title for PubMed "neighbors" (related citations) of the original publications. It then requires specific words in the resulting rows, sorts alphabetically by author name and title, aligns the columns, and truncates the lines for easier viewing: 2960822 Anton IA A eukaryotic repressor protein, the qa-1S gene prod 5264137 Arroyo-Begovich A In vitro formation of an active multienzyme complex 14942736 BONNER DM Gene-enzyme relationships in Neurospora. 5361218 Caroline DF Pyrimidine synthesis in Neurospora crassa: gene-enz 123642 Case ME Genetic evidence on the organization and action of ... EDirect will run on UNIX and Macintosh computers that have the Perl language installed, and under the Cygwin UNIX-emulation environment on Windows PCs. Documentation for EDirect is on the web at: http://www.ncbi.nlm.nih.gov/books/NBK179288 Questions or comments on EDirect may be sent to eutilities@ncbi.nlm.nih.gov. edirect/reorder-columns0000755000725300123250000000025512455536307015566 0ustar coremakecppcore#!/bin/sh # Usage: reorder-columns COLUMN NUMBERS... cmd="" com="$" for col in "$@" do cmd=`echo "$cmd$com$col"` com=", $" done awk -F '\t' -v 'OFS=\t' "{print $cmd}" edirect/epost0000755000725300123250000000040612122445476013573 0ustar coremakecppcore#!/bin/sh dir=`dirname "$0"` case "`uname -s`" in CYGWIN_NT*) # Use a negative match here because the shell treats 0 as success. if perl -e 'exit $^O !~ /^MSWin/'; then dir=`cygpath -w "$dir"` fi ;; esac exec perl "$dir"/edirect.pl -post "$@" edirect/elink0000755000725300123250000000040612122445476013543 0ustar coremakecppcore#!/bin/sh dir=`dirname "$0"` case "`uname -s`" in CYGWIN_NT*) # Use a negative match here because the shell treats 0 as success. if perl -e 'exit $^O !~ /^MSWin/'; then dir=`cygpath -w "$dir"` fi ;; esac exec perl "$dir"/edirect.pl -link "$@" edirect/sort-uniq-count0000755000725300123250000000031013004511271015505 0ustar coremakecppcore#!/bin/sh flags="f" if [ -n "$*" ] then flags=`echo " $*" | sed 's/[^bfinrs]//g'` if [ -z "$flags" ] then flags="s" fi fi sort "-$flags" | uniq -i -c | perl -pe 's/\s*(\d+)\s(.+)/$1\t$2/' edirect/entrez-phrase-search0000755000725300123250000000670012624142441016467 0ustar coremakecppcore#!/bin/bash -norc # simulates phrase search by queries on word pairs that may be indexed in specified field do_one_query() { esearch -db "$database" -query "$*" < /dev/null | xtract -pattern ENTREZ_DIRECT -element Count } word_at_a_time() { sed 's/[^+a-zA-Z0-9]/ /g; s/^ *//' | tr 'A-Z' 'a-z' | fmt -w 1 } replace_stop_words() { while read line do case "$line" in a | about | again | all | almost | also | although | always | \ among | an | and | another | any | are | as | at | be | \ because | been | before | being | between | both | but | by | \ can | could | did | do | does | done | due | during | each | \ either | enough | especially | etc | for | found | from | \ further | had | has | have | having | here | how | however | \ i | if | in | into | is | it | its | itself | just | kg | km | \ made | mainly | make | may | mg | might | ml | mm | most | \ mostly | must | nearly | neither | no | nor | obtained | of | \ often | on | our | overall | perhaps | pmid | quite | rather | \ really | regarding | seem | seen | several | should | show | \ showed | shown | shows | significantly | since | so | some | \ such | than | that | the | their | theirs | them | then | \ there | therefore | these | they | this | those | through | \ thus | to | upon | use | used | using | various | very | was | \ we | were | what | when | which | while | with | within | \ without | would ) echo "+" ;; * ) echo "$line" ;; esac done } group_phrases() { uniq | paste -sd "," - | sed -e 's/^+//g' -e 's/+$//g' -e 's/,+,/+/g' -e 's/^,//g' -e 's/,$//g' -e 's/+/ /g' } word_pairs() { while read first rest do if [ -z "$rest" ] then echo "$first" continue fi prev=$first for curr in $rest do echo "$prev $curr" prev="$curr" done done } get_phrase_counts() { echo "$*" | word_at_a_time | replace_stop_words | group_phrases | fmt -w 1 | tr ',' ' ' | word_pairs | while read qry do counts=`do_one_query "\"$qry\" [$field]"` if [ $counts -gt 0 ] then echo "$counts $qry" fi done } make_phrase_search() { get_phrase_counts "$@" | while read counts qry do if [ $counts -gt 9 ] then echo "\"$qry\" [$field]" fi done | sort -u | tr '\n' '*' | sed -e 's/*$//g' -e 's/*/ AND /g' } database="pubmed" field="TIAB" mode="search" while [ $# -gt 0 ] do case "$1" in -h | -help | --help ) mode=help break ;; -db | -database ) database=$2 shift shift ;; -field ) field=$2 shift shift ;; -count | -counts ) mode=count shift ;; -string ) mode=string shift ;; -* ) exec >&2 echo "$0: Unrecognized option $1" mode=usage shift ;; * ) break ;; esac done if [ $# -eq 0 ] then mode=usage fi case "$mode" in help | usage ) echo "< 0; use constant true => 1; # xtract version number $version = "2.9999"; # initialize memory hash my %memory = (); # initialize synopsis variables my %synopsis_acc = (); my $synopsis_max = 0; # initial level is global my $initial_level = 7; # utility subroutines sub convert_http { my $str = shift (@_); $str =~ s/\&\;/\&/g; $str =~ s/\&apos\;/\'/g; $str =~ s/\>\;/\>/g; $str =~ s/\<\;/\ 1 ) { push (@working, $dec); } elsif ( $itm eq "INNER" and $indx > 1 and $indx < $sclr ) { push (@working, $dec); } elsif ( $itm eq "TAIL" and $indx == $sclr and $sclr > 1 ) { push (@working, $dec); # even and odd are based only on index value, not list length } elsif ( $itm eq "EVEN" and ( $indx % 2) == 0 ) { push (@working, $dec); } elsif ( $itm eq "ODD" and ( $indx % 2) == 1 ) { push (@working, $dec); } } elsif ( $itm =~ /^@(.+)/ ) { # match unqualified @attribute my $att = $1; @atts = ($line =~ /<\S+ ([^>]+)>/g); foreach $val (@atts) { if ( $val =~ /$att=\"([^\"]+)\"/ ) { $dec = convert_http($1); push (@working, $dec); } } } elsif ( $itm =~ /(.+)@(.+)/ ) { # match qualified tag@attribute my $tag = $1; my $att = $2; @atts = ($line =~ /<$tag ([^>]+)>/g); foreach $val (@atts) { if ( $val =~ /$att=\"([^\"]+)\"/ ) { $dec = convert_http($1); push (@working, $dec); } } } elsif ( $itm =~ /(.+)\/(.+)/ ) { # match parent/child by tracking depth level of tokens my $tag = $1; my $chd = $2; @vals = ($line =~ /<$tag(?:\s+.+?)?>(.+?)<\/$tag>/g); foreach $val (@vals) { my $lvl = 0; my @tokens = split (/(?<=>)(?=<)/, $val); foreach $tkn (@tokens) { if ( $lvl < 0 ) { # ignore remainder if level drops below zero } elsif ( $tkn =~ /^<.+?>.+?<\/.+?>$/ ) { # content-containing tag does not change level if ( $tkn =~ /<$chd(?:\s+.+?)?>(.+?)<\/$chd>/ ) { # child tag matches, only accept if immediately below parent if ( $lvl == 0 ) { $dec = convert_http($1); push (@working, $dec); } } } elsif ( $tkn =~ /\/>$/ ) { # self-closing token does not change level } elsif ( $tkn =~ /^<[^\/]/ ) { # open tag increments level $lvl++; } elsif ( $tkn =~ /^<\// ) { # close tag decrements level $lvl--; } } } } elsif ( $itm =~ /(.+)\((.*)\)/ ) { # match self-closing tag (alias to print is in parentheses) my $tag = $1; my $rep = $2; if ( $rep eq "" ) { $rep = $tag; } # [ is it possible to look for both forms in one pass ? ] @vals = ($line =~ /(<$tag.*?><\/$tag>)/g); if ( scalar @vals != 0 ) { # found alternative form ( ) of self-closing tag foreach $val (@vals) { $dec = convert_http($rep); push (@working, $dec); } } else { # look for compact form ( ) if no match to alternative form if ( scalar @vals == 0 ) { @vals = ($line =~ /(<$tag.*?\/>)/g); foreach $val (@vals) { $dec = convert_http($rep); push (@working, $dec); } } } } else { # match tag with contents my $tag = $itm; @vals = ($line =~ /<$tag(?:\s+.+?)?>(.+?)<\/$tag>/g); foreach $val (@vals) { $dec = convert_http($val); push (@working, $dec); } } my $num = scalar @working; # if -first, -last, etc., remove all but matching subset if ( $num < 1 ) { # no matches collected, skip } elsif ( $cmmd eq "-first" ) { @working = splice (@working, 0, 1); $num = scalar @working; } elsif ( $cmmd eq "-last" ) { @working = splice (@working, $num - 1, 1); $num = scalar @working; } elsif ( $cmmd eq "-single" ) { if ( $num > 1 ) { $num = 0; } } elsif ( $cmmd eq "-head" ) { if ( $num < 2 ) { $num = 0; } else { @working = splice (@working, 0, 1); $num = scalar @working; } } elsif ( $cmmd eq "-inner" ) { if ( $num < 3 ) { $num = 0; } else { @working = splice (@working, 1, $num - 2); $num = scalar @working; } } elsif ( $cmmd eq "-tail" ) { if ( $num < 2 ) { $num = 0; } else { @working = splice (@working, $num - 1, 1); $num = scalar @working; } } # store current results into accumulator if ( $num == 0 ) { # no printable results } elsif ( $do_count ) { push (@accum, $num); } elsif ( $do_length ) { my $len = 0; foreach $dec (@working) { $len += length ( $dec ); } push (@accum, $len); } elsif ( $cmmd eq "-even" ) { my $take = false; foreach $dec (@working) { if ( $take ) { push (@accum, $dec); } $take = (! $take); } } elsif ( $cmmd eq "-odd" ) { my $take = true; foreach $dec (@working) { if ( $take ) { push (@accum, $dec); } $take = (! $take); } } else { foreach $dec (@working) { push (@accum, $dec); } } } # process results if ( scalar @accum == 0 ) { return false; } elsif ( $variable ne "") { # store accumulated components into variable my $str = $pfx; my $between = ""; foreach $dec (@accum) { $str .= "$between"; $between = $sep; $str .= "$dec"; } $str .= "$sfx"; $memory{"$variable"} = "$str"; } else { # format accumulated elements my $before = $prev . $pfx; my $between = ""; my $after = ""; foreach $dec (@accum) { print "$before"; $before = ""; print "$between"; $between = $sep; $after = $sfx; print "$dec"; } print "$after"; return true; } return false; } sub process_flags { my $line = shift (@_); my @args = @{shift (@_)}; my $tab = shift (@_); my $ret = shift (@_); my $indx = shift (@_); my $sclr = shift (@_); my $pfx = ""; my $sfx = ""; my $sep = "\t"; my $col = "\t"; my $lin = "\n"; my $okay = false; my $cmmd = ""; my $max = scalar @args; for ( my $i = 0; $i < $max; $i++ ) { # read command-line arguments my $val = $args[$i]; # if new argument starts with hyphen if ( $val =~ /^\-/ ) { # not an -element value, so set $okay flag to false $okay = false; # all flags should be followed by at least one value if ( $i + 1 >= $max and $val ne "-outline" and $val ne "-synopsis" ) { # warn that the last flag has no value print STDERR "No argument after '$val'\n"; # break out of for loop to return from function last; # otherwise safe to increment with $i++ and then dereference $args [$i] } } # print outline of XML structure if ( $val eq "-outline" ) { my $lvl = 0; my $dpth = 0; my @tokens = split (/(?<=>)(?=<)/, $line); foreach $tkn (@tokens) { if ( $lvl < 0 ) { # ignore remainder if level drops below zero } elsif ( $tkn =~ /^<.+?>.+?<\/(.+?)>$/ ) { # content-containing tag if ( $tkn =~ /^<(\S+).*?>.*>$/ ) { $tkn = $1; $lvl++; $dpth++; for ( $i = 0; $i < $dpth; $i++ ) { print " "; } print "$tkn\n"; $lvl--; $dpth--; } } elsif ( $tkn =~ /^<.+?\/>$/ ) { # self-closing token if ( $tkn =~ /^<(\S+).*?\/>$/ ) { $tkn = $1; $lvl++; $dpth++; for ( $i = 0; $i < $dpth; $i++ ) { print " "; } print "$tkn\n"; $lvl--; $dpth--; } } elsif ( $tkn =~ /^<[^\/]/ ) { # open tag increments level $lvl++; $dpth++; if ( $tkn =~ /^<(\S+).*?>$/ ) { $tkn = $1; if ( $tkn ne "?xml" and $tkn ne "!DOCTYPE" and $tkn ne "eSummaryResult" and $tkn ne "eLinkResult" and $tkn ne "eInfoResult" and $tkn ne "PubmedArticleSet" and $tkn ne "DocumentSummarySet" and $tkn ne "INSDSet" and $tkn ne "Entrezgene-Set" and $tkn ne "TaxaSet" ) { for ( $i = 0; $i < $dpth; $i++ ) { print " "; } print "$tkn\n"; } else { $dpth--; } } } elsif ( $tkn =~ /^<\// ) { # close tag decrements level $lvl--; $dpth--; } } # collect summary of XML paths, print at end } elsif ( $val eq "-synopsis" ) { my @arr = (); my $lvl = 0; my @tokens = split (/(?<=>)(?=<)/, $line); foreach $tkn (@tokens) { my $record_level = 0; if ( $lvl < 0 ) { # ignore remainder if level drops below zero } elsif ( $tkn =~ /^<.+?>.+?<\/(.+?)>$/ ) { # content-containing tag if ( $tkn =~ /^<(\S+).*?>.*>$/ ) { $tkn = $1; $lvl++; $arr[$lvl] = $tkn; $record_level = $lvl; $lvl--; } } elsif ( $tkn =~ /^<.+?\/>$/ ) { # self-closing token if ( $tkn =~ /^<(\S+).*?\/>$/ ) { $tkn = $1; $lvl++; $arr[$lvl] = $tkn; $record_level = $lvl; $lvl--; } } elsif ( $tkn =~ /^<[^\/]/ ) { # open tag increments level $lvl++; if ( $tkn =~ /^<(\S+).*?>$/ ) { $tkn = $1; $arr[$lvl] = $tkn; $record_level = $lvl; } } elsif ( $tkn =~ /^<\// ) { # close tag decrements level $lvl--; } if ( $record_level > 0 ) { my $str = ""; my $pfx = ""; for ( $i = 1; $i <= $record_level; $i++ ) { $itm = $arr[$i]; if ( $itm ne "?xml" and $itm ne "!DOCTYPE" and $itm ne "eSummaryResult" and $itm ne "eLinkResult" and $itm ne "eInfoResult" and $itm ne "PubmedArticleSet" and $itm ne "DocumentSummarySet" and $itm ne "INSDSet" and $itm ne "Entrezgene-Set" and $itm ne "TaxaSet" ) { $str .= "$pfx"; $pfx = "/"; $str .= "$arr[$i]"; } } if ( $str ne "" ) { my $val = 0; if ( exists ($synopsis_acc{"$str"}) ) { $val = $synopsis_acc{"$str"}; } $val++; $synopsis_acc{"$str"} = $val; if ( $val > $synopsis_max ) { $synopsis_max = $val; } print "$str\n" } } } # look for arguments that adjust output formatting } elsif ( $val eq "-pfx" ) { $i++; $pfx = convert_slash ( $args[$i] ); } elsif ( $val eq "-sfx" ) { $i++; $sfx = convert_slash ( $args[$i] ); } elsif ( $val eq "-sep" ) { $i++; $sep = convert_slash ( $args[$i] ); } elsif ( $val eq "-tab" ) { $i++; $col = convert_slash ( $args[$i] ); } elsif ( $val eq "-ret" ) { $i++; $lin = convert_slash ( $args[$i] ); } elsif ( $val eq "-lbl" ) { $i++; print "$tab"; $lbl = convert_slash ( $args[$i] ); print "$lbl"; $tab = $col; $ret = $lin; # look for -element or limiting variants, non-hyphenated arguments to follow } elsif ( $val eq "-element" or $val eq "-first" or $val eq "-last" or $val eq "-even" or $val eq "-odd" or $val eq "-single" or $val eq "-head" or $val eq "-inner" or $val eq "-tail" ) { $okay = true; $cmmd = $val; # hyphen followed by capital letters or digits is a variable } elsif ( $val =~ /^\-([A-Z0-9]+)([a-z]?)$/ ) { my $variable = $1; my $first_or_last = $2; $cmmd = "-element"; # detect f, l, etc., variable suffix for -first, -last, etc. (undocumented) if ( $first_or_last eq "f" ) { $cmmd = "-first"; } elsif ( $first_or_last eq "l" ) { $cmmd = "-last"; } elsif ( $first_or_last eq "e" ) { $cmmd = "-even"; } elsif ( $first_or_last eq "o" ) { $cmmd = "-odd"; } elsif ( $first_or_last eq "s" ) { $cmmd = "-single"; } elsif ( $first_or_last eq "h" ) { $cmmd = "-head"; } elsif ( $first_or_last eq "i" ) { $cmmd = "-inner"; } elsif ( $first_or_last eq "t" ) { $cmmd = "-tail"; } # clear the variable to avoid old value persisting if no subsequent match # commented out in favor of clearing by assigning a literal value containing an empty string # $memory{"$variable"} = ""; $i++; $val = $args[$i]; # set variable to new value if ( $val =~ /^\((.*)\)$/ ) { # parentheses contain literal value for variable $val = convert_slash ( $1 ); $memory{"$variable"} = "$val"; } else { # if XML tag name, record what would otherwise be printed by -element process_element ( $line, $val, $tab, $pfx, $sfx, $sep, $cmmd, $variable, $indx, $sclr ); } # reality check on unsupported arguments } elsif ( $val =~ /^\-/ ) { print STDERR "Unrecognized argument '$val'\n"; } elsif ( $okay ) { # process -element values if ( process_element ( $line, $val, $tab, $pfx, $sfx, $sep, $cmmd, "", $indx, $sclr )) { $tab = $col; $ret = $lin; } } else { print STDERR "No -element before '$val'\n"; } } return $tab, $ret; } sub has_content { my $line = shift (@_); my $str = shift (@_); my $indx = shift (@_); my $sclr = shift (@_); if ( $str eq "" or $str eq "&" ) { return false; } # match by non-empty variable value (undocumented) if ( $str =~ /^&([A-Z0-9]+)$/ ) { $str = $1; if ( exists ($memory{"$str"}) and $memory{"$str"} ne "" ) { return true; } return false; } # remove leading backslash used to prevent content from being mistaken for variable $str =~ s/^\\&/&/; # match by exploration flag (undocumented) if ( $str =~ /^\?([A-Z]+)/ ) { $str = $1; # first and last elements are always present, and are the same for a one-element list if ( $str eq "FIRST" and $indx == 1 ) { return true; } elsif ( $str eq "LAST" and $indx == $sclr ) { return true; # single, head, inner, tail are mutually exclusive situations } elsif ( $str eq "SINGLE" and $sclr == 1 ) { return true; } elsif ( $str eq "HEAD" and $indx == 1 and $sclr > 1 ) { return true; } elsif ( $str eq "INNER" and $indx > 1 and $indx < $sclr ) { return true; } elsif ( $str eq "TAIL" and $indx == $sclr and $sclr > 1 ) { return true; # even and odd are based only on index value, not list length } elsif ( $str eq "EVEN" and ( $indx % 2) == 0 ) { return true; } elsif ( $str eq "ODD" and ( $indx % 2) == 1 ) { return true; } } # remove leading backslash used to prevent content from being mistaken for flag $str =~ s/^\\\?/?/; # exact match by data contents, not using regular expression, case insensitive if ( index ( uc($line), uc($str) ) >= 0 ) { return true; } return false; } sub has_element { my $line = shift (@_); my $str = shift (@_); if ( $str eq "" ) { return false; } # track depth level of tokens if pattern was parent/child (undocumented) if ( $str =~ /(.+)\/(.+)/ ) { my $ptrn = $1; my $chld = $2; my @vals = (); my $lvl = 0; my $dpth = 0; my $in_pat = false; my @tokens = split (/(?<=>)(?=<)/, $line); foreach $tkn (@tokens) { if ( $lvl < 0 ) { # ignore remainder if level drops below zero } elsif ( $tkn =~ /^<.+?>.+?<\/.+?>$/ ) { # content-containing tag does not change level if ( $in_pat and $lvl == $dpth ) { push (@vals, $tkn); } } elsif ( $tkn =~ /\/>$/ ) { # self-closing token does not change level if ( $in_pat and $lvl == $dpth ) { push (@vals, $tkn); } } elsif ( $tkn =~ /^<[^\/]/ ) { # open tag increments level $lvl++; if ( $tkn =~ /<$ptrn(?:\s+.+?)?>/ and $dpth == 0 ) { $in_pat = true; $dpth = $lvl; push (@vals, $tkn); } } elsif ( $tkn =~ /^<\// ) { # close tag decrements level if ( $in_pat and $tkn =~ /<\/$ptrn>/ and $lvl == $dpth ) { push (@vals, $tkn); $in_pat = false; $dpth = 0; } $lvl--; } } if ( scalar @vals > 0 ) { my $sub = join ("", @vals); if ( has_element ( $sub, $chld )) { return true; } } return false; } # match by specific variable value (undocumented) if ( $str =~ /^\&([A-Z0-9]+)\:(.+)$/ ) { my $ptrn = $1; my $chld = $2; if ( exists ($memory{"$ptrn"}) and $memory{"$ptrn"} eq "$chld" ) { return true; } return false; } # match by non-empty variable value (undocumented) if ( $str =~ /^\&([A-Z0-9]+)$/ ) { $str = $1; if ( exists ($memory{"$str"}) and $memory{"$str"} ne "" ) { return true; } return false; } # regular expression search for element@attribute:value if ( $str =~ /(.+)@(.+)\:(.+)/ ) { my $ptrn = $1; my $attr = $2; my $chld = $3; if ( $line =~ /<$ptrn ([^>]+)>/i ) { my $atts = $1; if ( $atts =~ /$attr=\"$chld\"/ ) { return true; } } return false; } # regular expression search for @attribute:value if ( $str =~ /@(.+)\:(.+)/ ) { my $attr = $1; my $chld = $2; if ( $line =~ /<\S+ ([^>]+)>/i ) { my $atts = $1; if ( $atts =~ /$attr=\"$chld\"/ ) { return true; } } return false; } # regular expression search for element@attribute if ( $str =~ /(.+)@(.+)/ ) { my $ptrn = $1; my $attr = $2; if ( $line =~ /<$ptrn ([^>]+)>/i ) { my $atts = $1; if ( $atts =~ /$attr=\"([^\"]+)\"/ ) { return true; } } return false; } # regular expression search for @attribute if ( $str =~ /@(.+)/ ) { my $attr = $1; if ( $line =~ /<\S+ ([^>]+)>/i ) { my $atts = $1; if ( $atts =~ /$attr=\"([^\"]+)\"/ ) { return true; } } return false; } # INSDSeq special cases - &Feature:name and &Qualifier:name if ( $str =~ /^\&Feature\:(.+)/ ) { $str = "INSDFeature_key:" . "$1"; } elsif ( $str =~ /^\&Qualifier\:(.+)/ ) { $str = "INSDQualifier_name:" . "$1"; } # regular expression search for element:value if ( $str =~ /(.+)\:(.+)/ ) { my $ptrn = $1; my $chld = $2; if ( $line =~ /<$ptrn(?:\s+.+?)?>$chld<\/$ptrn>/i ) { return true; } return false; } # regular expression search for element tag if ( $line =~ /<$str(?:\s+.+?)?>/i ) { return true; } # also test self-closing tag if ( $line =~ /<$str(?:\s+.+?)?\/>/i ) { return true; } return false; } sub process_level { my $line = shift (@_); my $par = shift (@_); my $chd = shift (@_); my @args = @{shift (@_)}; my $tab = shift (@_); my $ret = shift (@_); my $level = shift (@_); my $indx = shift (@_); my $sclr = shift (@_); # handle heterogeneous child blocks if pattern was parent/* if ( $chd eq "*" and $par ne "" and $par ne "*" ) { my @vals = (); my $lvl = 0; my $in_pat = false; my $pfx = ""; my @working = (); if ( $line =~ /^<$par(?:\s+.+?)?>(.+?)<\/$par>$/ ) { $line = $1; } my @tokens = split (/(?<=>)(?=<)/, $line); foreach $tkn (@tokens) { if ( $lvl < 0 ) { # ignore remainder if level drops below zero } elsif ( $tkn =~ /^<.+?>.+?<\/.+?>$/ ) { # content-containing tag does not change level if ( $lvl > 0 ) { push (@working, $tkn); } else { # match to full object, push contents directly to final array push (@vals, $tkn); } } elsif ( $tkn =~ /\/>$/ ) { # self-closing token does not change level if ( $lvl > 0 ) { push (@working, $tkn); } else { push (@vals, $tkn); } } elsif ( $tkn =~ /^<[^\/]/ ) { # open tag increments level push (@working, $tkn); $lvl++; } elsif ( $tkn =~ /^<\// ) { # close tag decrements level push (@working, $tkn); $lvl--; if ( $lvl == 0 ) { my $str = join ("", @working); push (@vals, $str); @working = (); } } } if ( scalar @working > 0 ) { my $str = join ("", @working); push (@vals, $str); @working = (); } foreach $val (@vals) { if ( $level == $initial_level ) { print "$pfx"; %memory = (); } ( $tab, $ret ) = process_level ( $val, "", "", \@args, $tab, $ret, $level, $indx, $sclr ); if ( $level == $initial_level ) { $pfx = $ret; $tab = ""; } } return $tab, $ret; } # track depth level of tokens if pattern was parent/child if ( $chd ne "" ) { my @vals = (); my $lvl = 0; my $dpth = 0; my $in_pat = false; my $pfx = ""; my @working = (); my @tokens = split (/(?<=>)(?=<)/, $line); foreach $tkn (@tokens) { if ( $lvl < 0 ) { # ignore remainder if level drops below zero } elsif ( $tkn =~ /^<.+?>.+?<\/.+?>$/ ) { # content-containing tag does not change level if ( $in_pat ) { push (@working, $tkn); } elsif ( $tkn =~ /<$chd(?:\s+.+?)?>(.+?)<\/$chd>/ and $lvl < 2 ) { # match to full object, push directly to final array push (@vals, $tkn); } } elsif ( $tkn =~ /\/>$/ ) { # self-closing token does not change level if ( $in_pat ) { push (@working, $tkn); } } elsif ( $tkn =~ /^<[^\/]/ ) { # open tag increments level if ( $in_pat ) { push (@working, $tkn); } elsif ( $tkn =~ /<$chd(?:\s+.+?)?>/ and $lvl < 2 ) { $in_pat = true; $dpth = $lvl; push (@working, $tkn); } $lvl++; } elsif ( $tkn =~ /^<\// ) { # close tag decrements level $lvl--; if ( $in_pat ) { push (@working, $tkn); } if ( $tkn =~ /<\/$chd>/ ) { if ( $lvl <= $dpth ) { $in_pat = false; my $str = join ("", @working); push (@vals, $str); @working = (); } } } } if ( scalar @working > 0 ) { my $str = join ("", @working); push (@vals, $str); @working = (); } foreach $val (@vals) { if ( $level == $initial_level ) { print "$pfx"; %memory = (); } ( $tab, $ret ) = process_level ( $val, "", "", \@args, $tab, $ret, $level, $indx, $sclr ); if ( $level == $initial_level ) { $pfx = $ret; $tab = ""; } } return $tab, $ret; } my $max = scalar @args; # allow conditional arguments in any order my $go_on = true; while ( $go_on ) { if ( $max > 2 and $args[0] eq "-position" ) { # -position command filters by object position in list my $pstn = $args[1]; $max -= 2; @args = splice (@args, 2, $max); # first and last elements are always present, and are the same for a one-element list if ( $pstn eq "first" and $indx > 1 ) { return $tab, $ret; } elsif ( $pstn eq "last" and $indx < $sclr ) { return $tab, $ret; # single, head, inner, tail are mutually exclusive situations } elsif ( $pstn eq "single" and $sclr > 1 ) { return $tab, $ret; } elsif ( $pstn eq "head" and ( $indx > 1 or $sclr == 1 ) ) { return $tab, $ret; } elsif ( $pstn eq "inner" and ( $indx == 1 or $indx == $sclr ) ) { return $tab, $ret; } elsif ( $pstn eq "tail" and ( $indx < $sclr or $sclr == 1 ) ) { return $tab, $ret; # single and multiple are mutually exclusive situations } elsif ( $pstn eq "multiple" and $sclr == 1 ) { return $tab, $ret; # even and odd are based only on index value, not list length } elsif ( $pstn eq "even" and ( $indx % 2) == 1 ) { return $tab, $ret; } elsif ( $pstn eq "odd" and ( $indx % 2) == 0 ) { return $tab, $ret; # value is actual numeric index } elsif ( $pstn =~ /^[0-9]+$/ and $indx != $pstn ) { return $tab, $ret; } } elsif ( $max > 2 and $args[0] eq "-match" ) { # -match ... -and/-or ... command filters for indicated element tag [:value] my $required = 1; my $observed = 0; my $prevbool = ""; my $mtch = $args[1]; $max -= 2; @args = splice (@args, 2, $max); if ( has_element ( $line, $mtch )) { $observed++; } # "or" succeeds on any match, "and" requires all tests to match while ( $max > 2 and ( $args[0] eq "-or" or $args[0] eq "-and" )) { if ( $prevbool ne "" and $prevbool ne $args[0] ) { print STDERR "A mixture of -and and -or commands cannot follow -match\n"; return $tab, $ret; } $prevbool = $args[0]; if ( $prevbool eq "-and" ) { $required++; } $mtch = $args[1]; $max -= 2; @args = splice (@args, 2, $max); if ( has_element ( $line, $mtch )) { $observed++; } } if ( $observed < $required ) { return $tab, $ret; } } elsif ( $max > 2 and $args[0] eq "-avoid" ) { # -avoid ... -and ... command filters against indicated element tag [:value] my $skip = $args[1]; $max -= 2; @args = splice (@args, 2, $max); if ( has_element ( $line, $skip )) { return $tab, $ret; } # colloquial "and" is really logical "or" - any match bails out of function while ( $max > 2 and $args[0] eq "-and" ) { $skip = $args[1]; $max -= 2; @args = splice (@args, 2, $max); if ( has_element ( $line, $skip )) { return $tab, $ret; } } if ( $max > 1 and $args[0] eq "-or" ) { print STDERR "The -or command cannot follow -avoid\n"; return $tab, $ret; } } elsif ( $max > 2 and $args[0] eq "-present" ) { # -present ... -and/-or ... command filters for indicated data content my $required = 1; my $observed = 0; my $prevbool = ""; my $mtch = $args[1]; $max -= 2; @args = splice (@args, 2, $max); if ( has_content ( $line, $mtch, $indx, $sclr )) { $observed++; } # "or" succeeds on any match, "and" requires all tests to match while ( $max > 2 and ( $args[0] eq "-or" or $args[0] eq "-and" )) { if ( $prevbool ne "" and $prevbool ne $args[0] ) { print STDERR "A mixture of -and and -or commands cannot follow -present\n"; return $tab, $ret; } $prevbool = $args[0]; if ( $prevbool eq "-and" ) { $required++; } $mtch = $args[1]; $max -= 2; @args = splice (@args, 2, $max); if ( has_content ( $line, $mtch, $indx, $sclr )) { $observed++; } } if ( $observed < $required ) { return $tab, $ret; } } elsif ( $max > 2 and $args[0] eq "-absent" ) { # -absent ... -and ... command filters against indicated data content my $skip = $args[1]; $max -= 2; @args = splice (@args, 2, $max); if ( has_content ( $line, $skip, $indx, $sclr )) { return $tab, $ret; } # colloquial "and" is really logical "or" - any match bails out of function while ( $max > 2 and $args[0] eq "-and" ) { $skip = $args[1]; $max -= 2; @args = splice (@args, 2, $max); if ( has_content ( $line, $skip, $indx, $sclr )) { return $tab, $ret; } } if ( $max > 1 and $args[0] eq "-or" ) { print STDERR "The -or command cannot follow -absent\n"; return $tab, $ret; } } elsif ( $max > 1 and $args[0] eq "-trim" ) { # -trim removes first XML tag while ( $max > 1 and $args[0] eq "-trim" ) { $max--; @args = splice (@args, 1, $max); if ( $line =~ /^<.+?>(.+)$/ ) { $line = $1; } } } else { # no other conditional tests or trim commands, break loop $go_on = false; } } # at level 0 break recursion for nested organizers, process -element if ( $level < 1 ) { ( $tab, $ret ) = process_flags ( $line, \@args, $tab, $ret, $indx, $sclr ); return $tab, $ret; } # get name of command for current organizer level # initial capital signifies tokenized search matching nesting levels # (tokenizing is significantly slower than regular expression pattern matching) my $name = ""; my $capname = ""; if ( $level == 7 ) { $name = "-division"; $capname = "-Division"; } elsif ( $level == 6) { $name = "-group"; $capname = "-Group"; } elsif ( $level == 5) { $name = "-branch"; $capname = "-Branch"; } elsif ( $level == 4) { $name = "-block"; $capname = "-Block"; } elsif ( $level == 3) { $name = "-section"; $capname = "-Section"; } elsif ( $level == 2) { $name = "-subset"; $capname = "-Subset"; } elsif ( $level == 1) { $name = "-unit"; $capname = "-Unit"; } # find chains of arguments between indicated command names my $start = 0; my $stop = 0; for ( $start = 0; $start < $max; $start = $stop ) { $stop = $start + 1; while ( $stop < $max and $args[$stop] ne $name and $args[$stop] ne $capname ) { $stop++; } # collect arguments between -group, -block, or -subset directives my @tmp = @args; if ( $tmp[$start] eq $name and $stop - $start >= 2 ) { my $pat = $tmp[$start + 1]; my $chd = ""; if ( $pat =~ /\*\/(.+)/ ) { # -block */Taxon replaces -trim $pat = $1; if ( $line =~ /^<.+?>(.+)$/ ) { $line = $1; } } # INSDSeq special cases - &Feature and &Qualifier if ( $pat =~ /^\&Feature$/ ) { $pat = "INSDFeature"; } elsif ( $pat =~ /^\&Qualifier$/ ) { $pat = "INSDQualifier"; } if ( $pat =~ /(.+)\/(.+)/ ) { $pat = $1; $chd = $2; } $start += 2; my @group = splice (@tmp, $start, $stop - $start); # normal non-greedy pattern matching for non-nested XML my @vals = ($line =~ /(<$pat(?:\s+.+?)?>.+?<\/$pat>)/g); if ( scalar @vals > 0 ) { my $curr = 0; my $totl = scalar @vals; foreach $val (@vals) { $curr++; ( $tab, $ret ) = process_level ( $val, $pat, $chd, \@group, $tab, $ret, $level - 1, $curr, $totl ); } } else { @vals = ($line =~ /(<$pat(?:\s+.+?)\/>)/g); my $curr = 0; my $totl = scalar @vals; foreach $val (@vals) { $curr++; ( $tab, $ret ) = process_level ( $val, $pat, $chd, \@group, $tab, $ret, $level - 1, $curr, $totl ); } } } elsif ( $tmp[$start] eq $capname and $stop - $start >= 2 ) { my $pat = $tmp[$start + 1]; my $chd = ""; if ( $pat =~ /(.+)\/(.+)/ ) { $pat = $1; $chd = $2; } $start += 2; my @group = splice (@tmp, $start, $stop - $start); # tokenized pattern matching for XML with nested tags my @vals = (); my $lvl = 0; my $dpth = 0; my $in_pat = false; my @working = (); my @tokens = split (/(?<=>)(?=<)/, $line); foreach $tkn (@tokens) { if ( $lvl < 0 ) { # ignore remainder if level drops below zero } elsif ( $tkn =~ /^<.+?>.+?<\/.+?>$/ ) { # content-containing tag does not change level if ( $in_pat ) { push (@working, $tkn); } elsif ( $tkn =~ /<$pat(?:\s+.+?)?>(.+?)<\/$pat>/ ) { # match to full object, push directly to final array push (@vals, $tkn); } } elsif ( $tkn =~ /\/>$/ ) { # self-closing token does not change level if ( $in_pat ) { push (@working, $tkn); } } elsif ( $tkn =~ /^<[^\/]/ ) { # open tag increments level if ( $in_pat ) { push (@working, $tkn); } elsif ( $tkn =~ /<$pat(?:\s+.+?)?>/ ) { $in_pat = true; $dpth = $lvl; push (@working, $tkn); } $lvl++; } elsif ( $tkn =~ /^<\// ) { # close tag decrements level $lvl--; if ( $in_pat ) { push (@working, $tkn); } if ( $tkn =~ /<\/$pat>/ ) { if ( $lvl <= $dpth ) { $in_pat = false; my $str = join ("", @working); push (@vals, $str); @working = (); } } } } if ( scalar @working > 0 ) { my $str = join ("", @working); push (@vals, $str); @working = (); } my $curr = 0; my $totl = scalar @vals; foreach $val (@vals) { $curr++; ( $tab, $ret ) = process_level ( $val, $pat, $chd, \@group, $tab, $ret, $level - 1, $curr, $totl ); } } else { my @group = splice (@tmp, $start, $stop - $start); ( $tab, $ret ) = process_level ( $line, "", "", \@group, $tab, $ret, $level - 1, $indx, $sclr ); } } return $tab, $ret; } sub process_insd { # ... | xtract -insd [+/-] mat_peptide "%peptide" product peptide my $quot = shift (@_); my @args = @{shift (@_)}; my $max = scalar @args; my @working = (); # print common arguments push (@working, "-pattern"); push (@working, "INSDSeq"); push (@working, "-ACCN"); push (@working, "INSDSeq_accession-version"); # collect descriptors if ( $args[1] =~ /INSD/ ) { push (@working, "-pfx"); push (@working, "$quot\\n$quot"); push (@working, "-element"); push (@working, "$quot\&ACCN$quot"); for ( my $i = 1; $i < $max; $i++ ) { my $val = $args[$i]; push (@working, "-block"); push (@working, "INSDSeq"); push (@working, "-element"); push (@working, "$quot$val$quot"); } return @working; } # collect qualifiers my @qualifiers = (); my @required = (); my $feat = ""; my $partial = 0; for ( my $i = 1; $i < $max; $i++ ) { my $val = $args[$i]; if ( $feat eq "" ) { if ( $val eq "+" or $val eq "complete" ) { $partial = 1; } elsif ( $val eq "-" or $val eq "partial" ) { $partial = -1; } else { $feat = $val; } } else { if ( $val =~ /^\+(.+)/ ) { $val = $1; push (@required, $val); } push (@qualifiers, $val); } } push (@working, "-group"); push (@working, "INSDFeature"); my @itms = split (',', $feat); my $fcmd = "-match"; foreach $itm (@itms) { push (@working, "$fcmd"); push (@working, "INSDFeature_key:$itm"); $fcmd = "-or"; } if ( $partial == 1 ) { push (@working, "-avoid"); push (@working, "INSDFeature_partial5"); push (@working, "-and"); push (@working, "INSDFeature_partial3"); } elsif ( $partial == -1 ) { push (@working, "-match"); push (@working, "INSDFeature_partial5"); push (@working, "-or"); push (@working, "INSDFeature_partial3"); } my $rcmd = "-match"; foreach my $val (@required) { push (@working, "$rcmd"); push (@working, "INSDQualifier_name:$val"); $rcmd = "-and"; } push (@working, "-pfx"); push (@working, "$quot\\n$quot"); push (@working, "-element"); push (@working, "$quot&ACCN$quot"); foreach my $val (@qualifiers) { if ( $val =~ /INSD/ ) { push (@working, "-block"); push (@working, "INSDFeature"); push (@working, "-element"); push (@working, "$quot$val$quot"); } else { push (@working, "-block"); push (@working, "INSDQualifier"); if ( $val =~ /^%(.+)/ ) { $val = $1; push (@working, "-match"); push (@working, "INSDQualifier_name:$val"); push (@working, "-element"); push (@working, "$quot\%INSDQualifier_value$quot"); } else { push (@working, "-match"); push (@working, "INSDQualifier_name:$val"); push (@working, "-element"); push (@working, "INSDQualifier_value"); } } } return @working; } my $xtract_help = qq{ This Perl version of xtract is obsolete, and is no longer maintained or supported. The new version, which is written in the Go language, is more than an order of magnitude faster. To build the new xtract, download and install the open source Go compiler on your computer, then run: go build xtract.go }; sub xtract { # ... | xtract -pattern ... -group ... -block ... -subset ... -element ... my $compress_spaces = true; my $max = scalar @ARGV; if ( $max < 1 ) { print STDERR "No arguments were supplied to xtract.pl\n"; return; } my $timer = false; if ( $ARGV[0] eq "-timer" ) { # report execution time of original Perl implementation for comparison to new Go compiled version $timer = true; } elsif ( $ARGV[0] ne "-fallback" ) { # ensure that platform-specific Go compiled version was used if available print STDERR "\nPLEASE REWRITE YOUR SCRIPT TO CALL XTRACT INSTEAD OF XTRACT.PL.\n"; print STDERR "XTRACT WILL RUN AN IMPROVED, COMPILED EXECUTABLE THAT CAN USE\n"; print STDERR "MULTIPLE CPU CORES TO PROCESS RECORDS BETWEEN ONE AND TWO ORDERS\n"; print STDERR "OF MAGNITUDE FASTER THAN THE ORIGINAL PERL IMPLEMENTATION.\n\n"; return; } # skip past -timer or -fallback command shift @ARGV; $max = scalar @ARGV; if ( $max > 0 and $ARGV[0] eq "-help" ) { print "xtract $version (Perl version)\n"; print $xtract_help; return; } if ( $max > 0 and $ARGV[0] eq "-version" ) { print "$version\n"; return; } # report deprecated commands if ( $max > 0 ) { foreach $argument (@ARGV) { if ( $argument eq "-present" ) { print STDERR "Argument -present is deprecated, use -match Object:Value\n"; } if ( $argument eq "-absent" ) { print STDERR "Argument -absent is deprecated, use -avoid Object:Value\n"; } if ( $argument eq "-trim" ) { print STDERR "Argument -trim is deprecated, use -block */Object\n"; } if ( $argument eq "-make" or $argument eq "-fuse" or $argument eq "-split" or $argument eq "-pipe" or $argument eq "-repeat" ) { print STDERR "Argument '$argument' is deprecated\n"; } } } # -nocompress allows reading of BLAST XML match data with internal runs of spaces (undocumented) if ( $max > 0 and $ARGV[0] eq "-nocompress" ) { shift @ARGV; $max = scalar @ARGV; $compress_spaces = false; } # -insd to simplify extraction of INSDSeq qualifiers if ( $max > 0 and $ARGV[0] eq "-insd" ) { if ( $max < 3 and $ARGV[1] !~ /INSD/) { print STDERR "Must supply a feature key and at least one qualifier name\n"; return; } elsif ( -t STDIN ) { # -insd without piped input will generate an extraction script @ARGV = process_insd ( "\"", \@ARGV ); print "xtract "; foreach $itm (@ARGV) { print "$itm "; } print "| \\\n"; return; } else { # -insd with piped input will dynamically execute the extraction @ARGV = process_insd ( "", \@ARGV ); $max = scalar @ARGV; } } # recommended nested organizer levels are -group, -block, and -subset # intermediate levels (-division, -branch, -section, and -unit) kept in reserve # initial capitalized versions (-Group, -Block, etc.) use tokenized pattern matching if ( $max > 1 and $ARGV[0] eq "-split" ) { # -split reads data stream and writes chunks on separate lines my $pat = $ARGV[1]; my $in_pat = false; my @working = (); while ( defined($thisline = ) ) { $thisline =~ s/\r//g; $thisline =~ s/\n//g; $thisline =~ s/\t//g; if ( $compress_spaces ) { $thisline =~ s/ +/ /g; } $thisline =~ s/> +)(?=<)/, $thisline); foreach $tkn (@tokens) { if ( $tkn =~ /^ +(.+)$/ ) { $tkn = $1; } if ( $tkn =~ /^(.+) +$/ ) { $tkn = $1; } if ( $tkn =~ /<$pat(?:\s+.+?)?>(.+?)<\/$pat>/ ) { if ( scalar @working > 0 ) { my $str = join ("", @working); print "$str\n"; @working = (); } $in_pat = false; print "$tkn\n"; } elsif ( $tkn =~ /<$pat(?:\s+.+?)?>/ ) { push (@working, $tkn); $in_pat = true; } elsif ( $tkn =~ /<\/$pat>/ ) { push (@working, $tkn); $in_pat = false; my $str = join ("", @working); print "$str\n"; @working = (); } elsif ( $in_pat) { push (@working, $tkn); } } } if ( scalar @working > 0 ) { my $str = join ("", @working); print "$str\n"; } return; } if ( $max > 0 and $ARGV[0] eq "-format" ) { my $lvl = 0; my $xml_ok = true; my $doctype_ok = true; my $root_token = ""; my $print_head = false; my $print_tail = false; my $xml = ""; my $doctype = ""; my $pfx = ""; my $sfx = ""; my $go_on = true; # support -pfx, -sfx, -xml, and -doctype arguments for ( my $i = 1; $i < $max; $i++ ) { if ( $ARGV[$i] eq "-pfx" ) { $i++; $pfx = convert_slash ( $ARGV[$i] ); } elsif ( $ARGV[$i] eq "-sfx" ) { $i++; $sfx = convert_slash ( $ARGV[$i] ); } elsif ( $ARGV[$i] eq "-xml" ) { $i++; $xml = convert_slash ( $ARGV[$i] ); } elsif ( $ARGV[$i] eq "-doctype" ) { $i++; $doctype = convert_slash ( $ARGV[$i] ); } } while ( $go_on ) { $thisline = ""; if ( $xml ne "" ) { $thisline = $xml; $xml = ""; } elsif ( $doctype ne "" ) { $thisline = $doctype; $doctype = ""; } elsif ( $pfx ne "" ) { $thisline = $pfx; $pfx = ""; } elsif ( defined($thisline = ) ) { } elsif ( $sfx ne "" ) { $thisline = $sfx; $sfx = ""; } else { $go_on = false; } if ( $go_on and $thisline ne "" ) { $thisline =~ s/\r//g; $thisline =~ s/\n//g; $thisline =~ s/\t//g; if ( $compress_spaces ) { $thisline =~ s/ +/ /g; } $thisline =~ s/> +)(?=<)/, $thisline); foreach $tkn (@tokens) { if ( $tkn =~ /^ +(.+)$/ ) { $tkn = $1; } if ( $tkn =~ /^(.+) +$/ ) { $tkn = $1; } if ( $lvl < 0 ) { # ignore remainder if level drops below zero } elsif ( $tkn =~ /^<\?xml/ ) { if ( $xml_ok ) { # only print first xml line print "$tkn\n"; $xml_ok = false; } } elsif ( $tkn =~ /^$/ ) { # processing instruction $lvl++; for ( $i = 1; $i < $lvl; $i++ ) { print " "; } print "$tkn\n"; $lvl--; } elsif ( $tkn =~ /^<\!--.+-->$/ ) { # comment $lvl++; for ( $i = 1; $i < $lvl; $i++ ) { print " "; } print "$tkn\n"; $lvl--; } elsif ( $tkn =~ /^<.+?>.+?<\/(.+?)>$/ ) { # content-containing tag if ( $tkn =~ /^<(\S+).*?>.*>$/ ) { $lvl++; for ( $i = 1; $i < $lvl; $i++ ) { print " "; } print "$tkn\n"; $lvl--; } } elsif ( $tkn =~ /^<.+?\/>$/ ) { # self-closing token if ( $tkn =~ /^<(\S+).*?\/>$/ ) { $lvl++; for ( $i = 1; $i < $lvl; $i++ ) { print " "; } print "$tkn\n"; $lvl--; } } elsif ( $tkn =~ /^<[^\/]/ ) { # open tag increments level $lvl++; if ( $tkn eq "<$root_token>" ) { if ( $print_head ) { print "<$root_token>\n"; $print_head = false; $print_tail = true; } } else { if ( $tkn =~ /^<(\S+).*?>$/ ) { for ( $i = 1; $i < $lvl; $i++ ) { print " "; } print "$tkn\n"; } elsif ( $tkn =~ /^<(\S+).*?$/ ) { for ( $i = 1; $i < $lvl; $i++ ) { print " "; } print "$tkn"; } } } elsif ( $tkn =~ /^<\// ) { # close tag decrements level if ( $tkn ne "" ) { for ( $i = 1; $i < $lvl; $i++ ) { print " "; } print "$tkn\n"; } $lvl--; } elsif ( $tkn =~ /^>$/ ) { print "$tkn\n"; } elsif ( $tkn =~ />$/ ) { print " $tkn\n"; } else { print " $tkn"; } } } } if ( $print_tail ) { print "\n"; } return; } my $ret = ""; my $tab = ""; if ( $max > 2 and $ARGV[0] eq "-pipe" ) { # -pipe reads data that has already been compressed by xtract -split shift @ARGV; while ( defined($thisline = ) ) { $thisline =~ s/\r//g; $thisline =~ s/\n//g; $thisline =~ s/\t//g; if ( $compress_spaces ) { $thisline =~ s/ +/ /g; } $thisline =~ s/> + 2 and $ARGV[0] eq "-pattern" ) { # simple -pattern reads data stream and processes one pattern at a time my $pat = $ARGV[1]; my $chd = ""; if ( $pat =~ /(.+)\/(.+)/ ) { $pat = $1; $chd = $2; } @ARGV = splice (@ARGV, 2, $max - 2); my $in_pat = false; my @working = (); while ( defined($thisline = ) ) { $thisline =~ s/\r//g; $thisline =~ s/\n//g; $thisline =~ s/\t//g; if ( $compress_spaces ) { $thisline =~ s/ +/ /g; } $thisline =~ s/> +)(?=<)/, $thisline); foreach $tkn (@tokens) { if ( $tkn =~ /^ +(.+)$/ ) { $tkn = $1; } if ( $tkn =~ /^(.+) +$/ ) { $tkn = $1; } if ( $tkn =~ /<$pat(?:\s+.+?)?>/ ) { if ( $tkn =~ /<$pat(?:\s+.+?)?>(.+?)<\/$pat>/ ) { if ( scalar @working > 0 ) { my $str = join ("", @working); @working = (); my @tmp = @ARGV; %memory = (); ( $tab, $ret ) = process_level ( $str, $pat, $chd, \@tmp, "", "", $initial_level, 1, 1 ); print "$ret"; } $in_pat = false; my @tmp = @ARGV; %memory = (); ( $tab, $ret ) = process_level ( $tkn, $pat, $chd, \@tmp, "", "", $initial_level, 1, 1 ); print "$ret"; } else { push (@working, $tkn); $in_pat = true; } } elsif ( $tkn =~ /<\/$pat>/ ) { push (@working, $tkn); $in_pat = false; my $str = join ("", @working); @working = (); my @tmp = @ARGV; %memory = (); ( $tab, $ret ) = process_level ( $str, $pat, $chd, \@tmp, "", "", $initial_level, 1, 1 ); print "$ret"; } elsif ( $in_pat) { push (@working, $tkn); } } } if ( scalar @working > 0 ) { my $str = join ("", @working); @working = (); my @tmp = @ARGV; %memory = (); ( $tab, $ret ) = process_level ( $str, $pat, $chd, \@tmp, "", "", $initial_level, 1, 1 ); print "$ret"; } my $end_time = time(); my $elapsed = $end_time - $begin_time; if ( $timer ) { if ( $elapsed > 1 ) { print STDERR "\nXtract.pl ran in $elapsed seconds\n\n"; } elsif ( $elapsed > 0 ) { print STDERR "\nXtract.pl ran in $elapsed second\n\n"; } } return; } if ( $max > 2 and $ARGV[0] eq "-Pattern" ) { # capitalized -Pattern tracks depth of top-level pattern (e.g., for nested Taxon structure) my $pat = $ARGV[1]; my $chd = ""; if ( $pat =~ /(.+)\/(.+)/ ) { $pat = $1; $chd = $2; } @ARGV = splice (@ARGV, 2, $max - 2); my $in_pat = false; my $lvl = 0; my @working = (); while ( defined($thisline = ) ) { $thisline =~ s/\r//g; $thisline =~ s/\n//g; $thisline =~ s/\t//g; if ( $compress_spaces ) { $thisline =~ s/ +/ /g; } $thisline =~ s/> +)(?=<)/, $thisline); foreach $tkn (@tokens) { if ( $tkn =~ /^ +(.+)$/ ) { $tkn = $1; } if ( $tkn =~ /^(.+) +$/ ) { $tkn = $1; } if ( $tkn =~ /<$pat(?:\s+.+?)?>/ ) { if ( $tkn =~ /<$pat(?:\s+.+?)?>(.+?)<\/$pat>/ ) { if ( scalar @working > 0 ) { my $str = join ("", @working); @working = (); my @tmp = @ARGV; %memory = (); ( $tab, $ret ) = process_level ( $str, $pat, $chd, \@tmp, "", "", $initial_level, 1, 1 ); print "$ret"; } $in_pat = false; my @tmp = @ARGV; %memory = (); ( $tab, $ret ) = process_level ( $tkn, $pat, $chd, \@tmp, "", "", $initial_level, 1, 1 ); print "$ret"; } else { $lvl++; push (@working, $tkn); $in_pat = true; } } elsif ( $tkn =~ /<\/$pat>/ ) { $lvl--; if ($lvl < 1) { push (@working, $tkn); $in_pat = false; my $str = join ("", @working); @working = (); my @tmp = @ARGV; %memory = (); ( $tab, $ret ) = process_level ( $str, $pat, $chd, \@tmp, "", "", $initial_level, 1, 1 ); print "$ret"; } elsif ( $in_pat) { push (@working, $tkn); } } elsif ( $in_pat) { push (@working, $tkn); } } } if ( scalar @working > 0 ) { my $str = join ("", @working); @working = (); my @tmp = @ARGV; %memory = (); ( $tab, $ret ) = process_level ( $str, $pat, $chd, \@tmp, "", "", $initial_level, 1, 1 ); print "$ret"; } return; } my $rpt = 1; # read entire XML input stream into a single string $holdTerminator = $/; undef $/; $data = ; $/ = $holdTerminator; # remove newlines, tabs, space between tokens, compress runs of spaces, $data =~ s/\r//g; $data =~ s/\n//g; $data =~ s/\t//g; if ( $compress_spaces ) { $data =~ s/ +/ /g; } $data =~ s/> + 2 and $ARGV[0] eq "-repeat" ) { # first look for -repeat count for performance testing $rpt = $ARGV[1]; @ARGV = splice (@ARGV, 2, $max - 2); $max = scalar @ARGV; } if ( $ARGV[0] ne "-pattern" ) { # no -pattern, process entire XML without looking for pattern my @args = @ARGV; %memory = (); ( $tab, $ret ) = process_level ( $data, "", "", \@args, "", "", $initial_level, 1, 1 ); print "$ret"; return; } # split by -pattern - can handle single term or parent/child (including heterogeneous parent/*) my $ptrn = $ARGV[1]; my $chld = ""; if ( $ptrn =~ /(.+)\/(.+)/ ) { $ptrn = $1; $chld = $2; } @vals = ($data =~ /(<$ptrn(?:\s+.+?)?>.+?<\/$ptrn>)/g); if ( $max == 2 ) { foreach $val (@vals) { print "$val\n"; } return; } # process remaining arguments my @args = splice (@ARGV, 2, $max - 2); my $curr = 0; my $totl = scalar @vals; for ( my $idx = 0; $idx < $rpt; $idx++ ) { foreach $val (@vals) { %memory = (); ( $tab, $ret ) = process_level ( $val, $ptrn, $chld, \@args, "", "", $initial_level, $curr, $totl ); print "$ret"; } } } # execute XML extraction xtract (); # if -synopsis, print cumulative summary of XML paths # if ( scalar (keys %synopsis_acc) > 0 ) { # my @keys; # @keys = keys %synopsis_acc; # for my $ky (sort @keys) { # $vl = $synopsis_acc{$ky}; # for ( $i = 0; $i < $vl; $i++ ) { # print "$ky\n"; # } # } # } # close input and output files close (STDIN); close (STDOUT); close (STDERR); edirect/run-ncbi-converter0000755000725300123250000000512513037752411016162 0ustar coremakecppcore#!/usr/bin/perl -w use strict; use File::Path; use Net::FTP; use POSIX qw(uname); my $cache_dir = "$ENV{HOME}/.cache/ncbi-converters"; if (defined $ENV{NCBI_CONVERTER_DIR}) { $cache_dir = $ENV{NCBI_CONVERTER_DIR}; } my $server = 'ftp.ncbi.nlm.nih.gov'; my $platform = DetectPlatform(); my $dir = "/toolbox/ncbi_tools/converters/by_platform/$platform"; my $ext = ($platform eq 'win') ? 'zip' : 'gz'; my $binext = ($platform eq 'win') ? '.exe' : ''; my $archive = "$ARGV[0].$platform.$ext"; my $executable = "$cache_dir/" . $ARGV[0] . $binext; if ( ! -d $cache_dir ) { File::Path::make_path($cache_dir) or die "Unable to ensure the existence of $cache_dir: $!"; } my $ftp = new Net::FTP($server, Passive => 1) or die "Unable to connect to FTP server: $!"; $ftp->login or die "Unable to log in to FTP server"; $ftp->cwd($dir) or die "Unable to change to $dir"; $ftp->binary or warn "Unable to set binary mode"; my $time = $ftp->mdtm($archive); my @stats = stat "$cache_dir/$ARGV[0]$binext"; if ( !@stats || $stats[9] < $time) { $ftp->get("$archive", "$cache_dir/$archive") or die "Unable to download $archive"; utime $time, $time, "$cache_dir/$archive"; my $pid = fork(); if ($pid < 0) { die "Unable to fork for unpacking: $!"; } elsif ($pid > 0) { waitpid($pid, 0); chmod(0777 &~ umask, $executable); utime $time, $time, $executable; } else { chdir($cache_dir); if ($platform eq 'win') { exec('unzip', $archive); } else { system('gunzip', '-n', $archive); rename("$ARGV[0].$platform", $ARGV[0]); exit 0; } } } shift; exec($executable, @ARGV); sub DetectPlatform { my @uname = uname(); my $OS = $uname[0]; my $CPU = $uname[4]; my $pf; my $last_built; if ($OS =~ /^CYGWIN/) { $pf = 'win'; } elsif ($OS eq 'Darwin') { $pf = 'mac'; } elsif ($OS eq 'Linux') { if ($CPU =~ /i\d86/) { $pf = 'linux32'; $last_built = 'November 2014'; } elsif ($CPU eq 'x86_64') { $pf = 'linux64'; } } elsif ($OS eq 'SunOS') { if ($CPU =~ /^s/) { $pf = 'solaris'; $last_built = 'March 2014'; } else { $pf = 'solaris-x86'; $last_built = 'September 2014'; } } else { die "No prebuilt binaries available for $OS/$CPU"; } if (defined $last_built) { warn "NCBI no longer builds for $OS/$CPU; using a binary from" . " $last_built"; } return $pf; } edirect/amino-acid-composition0000755000725300123250000000101613003505115016763 0ustar coremakecppcore#!/bin/bash -norc abbrev=( Ala Asx Cys Asp Glu Phe Gly His Ile \ Xle Lys Leu Met Asn Pyl Pro Gln Arg \ Ser Thr Sec Val Trp Xxx Tyr Glx ) AminoAcidComp() { local count while read num lttr do idx=$(printf %i "'$lttr'") ofs=$((idx-97)) count[$ofs]="$num" done <<< "$1" for i in {0..25} do echo -e "${abbrev[$i]}\t${count[$i]-0}" done | sort } while read seq do comp="$(echo "$seq" | tr A-Z a-z | sed 's/[^a-z]//g' | fold -w 1 | sort-uniq-count)" AminoAcidComp "$comp" done edirect/edirect.pl0000755000725300123250000031527513040250272014473 0ustar coremakecppcore#!/usr/bin/perl # =========================================================================== # # PUBLIC DOMAIN NOTICE # National Center for Biotechnology Information (NCBI) # # This software/database is a "United States Government Work" under the # terms of the United States Copyright Act. It was written as part of # the author's official duties as a United States Government employee and # thus cannot be copyrighted. This software/database is freely available # to the public for use. The National Library of Medicine and the U.S. # Government do not place any restriction on its use or reproduction. # We would, however, appreciate having the NCBI and the author cited in # any work or product based on this material. # # Although all reasonable efforts have been taken to ensure the accuracy # and reliability of the software and data, the NLM and the U.S. # Government do not and cannot warrant the performance or results that # may be obtained by using this software or data. The NLM and the U.S. # Government disclaim all warranties, express or implied, including # warranties of performance, merchantability or fitness for any particular # purpose. # # =========================================================================== # # File Name: edirect.pl # # Author: Jonathan Kans # # Version Creation Date: 8/20/12 # # ========================================================================== # Entrez Direct - EDirect # use strict; use warnings; my ($LibDir, $ScriptName); use File::Spec; BEGIN { my $Volume; ($Volume, $LibDir, $ScriptName) = File::Spec->splitpath($0); $LibDir = File::Spec->catpath($Volume, $LibDir, ''); if (my $RealPathname = eval {readlink $0}) { do { $RealPathname = File::Spec->rel2abs($RealPathname, $LibDir); ($Volume, $LibDir, undef) = File::Spec->splitpath($RealPathname); $LibDir = File::Spec->catpath($Volume, $LibDir, '') } while ($RealPathname = eval {readlink $RealPathname}); } else { $LibDir = File::Spec->rel2abs($LibDir) } $LibDir .= '/aux/lib/perl5'; } use lib $LibDir; # usage - edirect.pl -function arguments use Data::Dumper; use Encode; use Getopt::Long; use HTML::Entities; use LWP::Simple; use LWP::UserAgent; use Net::hostent; use POSIX; use Time::HiRes; use URI::Escape; # required first argument is name of function to run $fnc = shift or die "Must supply function name on command line\n"; # get starting time $begin_time = Time::HiRes::time(); # definitions use constant false => 0; use constant true => 1; # EDirect version number $version = "6.10"; # URL address components $base = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"; $ecitmat = "ecitmatch.cgi"; $efetch = "efetch.fcgi"; $einfo = "einfo.fcgi"; $elink = "elink.fcgi"; $epost = "epost.fcgi"; $esearch = "esearch.fcgi"; $espell = "espell.fcgi"; $esummary = "esummary.fcgi"; # utility subroutines sub clearflags { @rest = (); %labels = (); %macros = (); $alias = ""; $author = ""; $basx = ""; $batch = false; $chr_start = -1; $chr_stop = -1; $clean = false; $cmd = ""; $complexity = 0; $db = ""; $dbase = ""; $dbs = ""; $dbto = ""; $debug = false; $drop = false; $dttype = ""; $emaddr = ""; $email = ""; $err = ""; $extend = -1; $extrafeat = -1; $field = ""; $fields = false; $feature = ""; $gtype = ""; $help = false; $holding = ""; $http = ""; $id = ""; $input = ""; $journal = ""; $just_num = false; $key = ""; $lbl = ""; $links = false; $location = ""; $log = false; $max = 0; $meadow = ""; $min = 0; $mndate = ""; $mode = ""; $molecule = ""; $mxdate = ""; $name = ""; $neighbor = false; $num = ""; $organism = ""; $output = ""; $page = ""; $pair = ""; $pipe = false; $pub = ""; $query = ""; $related = false; $rldate = 0; $seq_start = 0; $seq_stop = 0; $silent = false; $sort = ""; $source = ""; $spell = false; $status = ""; $stp = ""; $strand = ""; $tool = ""; $trim = false; $trunc = false; $tuul = ""; $type = ""; $verbose = false; $volume = ""; $web = ""; $word = false; $year = ""; } # gets a live UID for any database sub get_zero_uid { my $db = shift (@_); my $val = ""; %zeroUidHash = ( 'assembly' => '443538', 'bioproject' => '146229', 'biosample' => '3737421', 'biosystems' => '1223165', 'blastdbinfo' => '1023214', 'books' => '1371014', 'cdd' => '277499', 'clinvar' => '10510', 'clone' => '18646800', 'dbvar' => '6173073', 'gap' => '872875', 'gds' => '200022309', 'gencoll' => '398148', 'gene' => '3667', 'genome' => '52', 'genomeprj' => '72363', 'geoprofiles' => '16029743', 'homologene' => '510', 'medgen' => '162753', 'mesh' => '68007328', 'nlmcatalog' => '0404511', 'nuccore' => '1322283', 'nucest' => '338968657', 'nucgss' => '168803471', 'nucleotide' => '1322283', 'omia' => '1920', 'omim' => '176730', 'pcassay' => '1901', 'pccompound' => '16132302', 'pcsubstance' => '126522451', 'pmc' => '209839', 'popset' => '27228303', 'probe' => '9997691', 'protein' => '4557671', 'proteinclusters' => '2945638', 'pubmed' => '2539356', 'pubmedhealth' => '27364', 'seqannot' => '9561', 'snp' => '137853337', 'sra' => '190091', 'structure' => '61024', 'taxonomy' => '562', 'unigene' => '1132160', ); if ( defined $zeroUidHash{$db} ) { $val = $zeroUidHash{$db}; } # return $val below to restore debugging test return 0; } # support for substitution of (#keyword) to full query phrase or URL component sub map_labels { my $qury = shift (@_); if ( $query !~ /\(#/ ) { return $qury; } if ( scalar (keys %labels) > 0 ) { for ( keys %labels ) { $ky = $_; $vl = $labels{$_}; $qury =~ s/\((#$ky)\)/\($vl#\)/g; } $qury =~ s/\((\w+)#\)/\(#$1\)/g; } return $qury; } sub map_macros { my $qury = shift (@_); if ( $qury !~ /\(#/ ) { return $qury; } if ( scalar (keys %macros) > 0 ) { for ( keys %macros ) { $ky = $_; $vl = $macros{$_}; $qury =~ s/\((\#$ky)\)/$vl/g; } } return $qury; } sub read_aliases { if ( $alias ne "" ) { if (open (my $PROXY_IN, $alias)) { while ( $thisline = <$PROXY_IN> ) { $thisline =~ s/\r//; $thisline =~ s/\n//; $thisline =~ s/ +/ /g; $thisline =~ s/> name =~ /\./) { $rhs = $h->name; } } } $addr = $lhs . '@' . $rhs; } return $addr; } # correct misspellings in query sub spell_check_query { my $db = shift (@_); my $qury = shift (@_); my $url = $base . $espell; my $enc = uri_escape($query); $arg = "db=$db&term=$enc"; my $data = do_post ($url, $arg, $tool, $email, true); Encode::_utf8_on($data); $qury = $1 if ( $data =~ /(.+)<\/CorrectedQuery>/ ); return $qury; } # elink and epost currently need a separate ESearch to get the correct result count sub get_count { my $dbsx = shift (@_); my $webx = shift (@_); my $keyx = shift (@_); my $tulx = shift (@_); my $emlx = shift (@_); test_edirect ( $dbsx, $webx, $keyx, "1", "count" ); $url = $base . $esearch; $url .= "?db=$dbsx&query_key=$keyx&WebEnv=$webx"; $url .= "&retmax=0&usehistory=y"; $url .= "&edirect=$version"; if ( $tulx eq "" ) { $tulx = "entrez-direct"; } if ( $tulx ne "" ) { $tulx =~ s/\s+/\+/g; $url .= "&tool=$tulx-count"; } if ( $emlx eq "" ) { $emlx = get_email (); } if ( $emlx ne "" ) { $emlx =~ s/\s+/\+/g; $url .= "&email=$emlx"; } $keyx = ""; $numx = ""; $errx = ""; $output = get ($url); if ( $output eq "" ) { print STDERR "No get_count output returned from '$url'\n"; } if ( $debug ) { print STDERR "$output\n"; } $keyx = $1 if ($output =~ /(\S+)<\/QueryKey>/); $numx = $1 if ($output =~ /(\S+)<\/Count>/); $errx = $1 if ($output =~ /(.+?)<\/Error>/i); if ( $errx ne "" ) { close (STDOUT); die "ERROR in count output: $errx\nURL: $url\n\n"; } if ( $numx eq "" ) { die "Count value not found in count output - WebEnv $webx\n"; } return $numx, $keyx; } sub get_uids { my @working = (); my $dbsx = shift (@_); my $webx = shift (@_); my $keyx = shift (@_); my $sttx = shift (@_); my $chkx = shift (@_); my $numx = shift (@_); my $tulx = shift (@_); my $emlx = shift (@_); test_edirect ( $dbsx, $webx, $keyx, $numx, "uids" ); # adjust retmax if -stop has been overridden # e.g., efetch -format uid -start 2 -stop 9 if ( $sttx + $chkx > $numx ) { $chkx = $numx - $sttx; } $url = $base . $esearch . "?db=$dbsx&query_key=$keyx&WebEnv=$webx"; $url .= "&rettype=uilist&retmode=text"; $url .= "&retstart=$sttx&retmax=$chkx"; $url .= "&edirect=$version"; if ( $tulx eq "" ) { $tulx = "edirect"; } if ( $tulx ne "" ) { $tulx =~ s/\s+/\+/g; $url .= "&tool=$tulx"; } if ( $emlx eq "" ) { $emlx = get_email (); } if ( $emlx ne "" ) { $emlx =~ s/\s+/\+/g; $url .= "&email=$emlx"; } if ( $debug ) { print STDERR "$url\n"; } $data = get ($url); if ( $data eq "" ) { print STDERR "No get_uids output returned from '$url'\n"; } if ( $debug ) { print STDERR "$data\n"; } my @ids = ($data =~ /(\d+)<\/Id>/g); foreach $uid (@ids) { push (@working, $uid); } return @working; } # send actual query sub do_post_yielding_ref { my $urlx = shift (@_); my $argx = shift (@_); my $tulx = shift (@_); my $emlx = shift (@_); my $intr = shift (@_); $argx .= "&edirect=$version"; if ( $intr ) { if ( $tulx eq "" ) { $tulx = "edirect"; } if ( $tulx ne "" ) { $tulx =~ s/\s+/\+/g; $argx .= "&tool=$tulx"; } if ( $emlx eq "" ) { $emlx = get_email (); } if ( $emlx ne "" ) { $emlx =~ s/\s+/\+/g; $argx .= "&email=$emlx"; } } my $empty = ''; $rslt = \$empty; if ( $debug or $log ) { print STDERR "$urlx?$argx\n"; } if ( $http eq "get" or $http eq "GET" ) { if ( $argx ne "" ) { $urlx .= "?"; $urlx .= "$argx"; } $rslt = get ($urlx); if ( $rslt eq "" ) { print STDERR "No do_get output returned from '$urlx'\n"; } if ( $debug ) { print STDERR "$rslt\n"; } return \$rslt; } $usragnt = new LWP::UserAgent (timeout => 300); $usragnt->env_proxy; $req = new HTTP::Request POST => "$urlx"; $req->content_type('application/x-www-form-urlencoded'); $req->content("$argx"); $res = $usragnt->request ( $req ); if ( $res->is_success) { $rslt = $res->content_ref; } else { print STDERR $res->status_line . "\n"; } if ( $$rslt eq "" ) { if ( $argx ne "" ) { $urlx .= "?"; $urlx .= "$argx"; } print STDERR "No do_post output returned from '$urlx'\n"; print STDERR "Result of do_post http request is\n"; print STDERR Dumper($res); print STDERR "\n"; } if ( $debug ) { print STDERR $$rslt, "\n"; } return $rslt; } sub do_post { my $rslt = do_post_yielding_ref(@_); return $$rslt; } # read ENTREZ_DIRECT data structure sub read_edirect { my $dbsx = ""; my $webx = ""; my $keyx = ""; my $numx = ""; my $stpx = ""; my $errx = ""; my $tulx = ""; my $emlx = ""; my $inlabel = false; my $inmacro = false; my $ky = ""; my $vl = ""; @other = (); $has_num = false; $all_num = true; while ( defined($thisline = ) ) { $thisline =~ s/\r//; $thisline =~ s/\n//; $thisline =~ s/^\s+//; $thisline =~ s/\s+$//; if ( $thisline =~ // ) { $inlabel = true; } elsif ( $thisline =~ /<\/Labels>/ ) { $inlabel = false; } elsif ( $thisline =~ // ) { $inmacro = true; } elsif ( $thisline =~ /<\/Macros>/ ) { $inmacro = false; } elsif ( $inlabel ) { if ( $thisline =~ /