Text-Ngrams-2.005/0000700000175000017500000000000012120322302012636 5ustar vladovladoText-Ngrams-2.005/Ngrams.pm0000755000175000017500000007321412110177337014466 0ustar vladovlado# (c) 2003-2013 Vlado Keselj http://web.cs.dal.ca/~vlado # # Text::Ngrams - A Perl module for N-grams processing package Text::Ngrams; use strict; require Exporter; use Carp; use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); @ISA = qw(Exporter); %EXPORT_TAGS = ( 'all' => [ qw(new encode_S decode_S) ] ); @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); @EXPORT = qw(new); $VERSION = '2.005'; use vars qw($Version $Revision); $Version = $VERSION; ($Revision = substr(q$Revision: 399 $, 10)) =~ s/\s+$//; use vars @EXPORT_OK; use vars qw(); # non-exported package globals go here sub new { my $package = shift; $package = ref($package) || $package; my $self = {}; my (%params) = @_; $self->{windowsize} = exists($params{windowsize}) ? $params{windowsize} : 3; die "nonpositive window size: $self->{windowsize}" unless $self->{windowsize} > 0; delete($params{windowsize}); if (! exists($params{type}) or $params{type} eq 'character') { $self->{skiprex} = ''; $self->{tokenrex} = qr/([a-zA-Z]|[^a-zA-Z]+)/; $self->{processtoken} = sub { s/[^a-zA-Z]+/ /; $_ = uc $_ }; $self->{allow_iproc} = 1; } elsif ($params{type} eq 'utf8') { $self->{skiprex} = ''; $self->{tokenrex} = qr/([\xF0-\xF4][\x80-\xBF][\x80-\xBF][\x80-\xBF] |[\xE0-\xEF][\x80-\xBF][\x80-\xBF] |[\xC2-\xDF][\x80-\xBF] |[\x00-\xFF])/x; $self->{processtoken} = ''; } #MJ -> #this type is analogous to the "character" type but defined for utf8 characters elsif ($params{type} eq 'utf8_character') { #$self->{inputlayer} #input layer to be put on the input stream by the function binmode before reading from a given stream #and to be removed by ***binmode HANDLE,":pop"*** after the reading from the particular stream is done #has to be a real layer (like ":encoding(utf8)"), not a pseudo layer (like ":utf8") #so that the psuedo layer ":pop" is able to remove this input layer $self->{inputlayer}=':encoding(utf8)'; #this will automatically decode input text from utf8 into Perl internal reporesentation of Unicode strings #and so the regular expressions for Unicode as well as the uc function can be performed on them $self->{skiprex} = ''; $self->{tokenrex} = qr/(\p{IsAlpha}|\P{IsAlpha}+)/; $self->{processtoken} = sub { s/\P{IsAlpha}+/ /; $_ = uc $_ ; $_=Encode::encode_utf8( $_ );}; #the last operation ***$_=Encode::encode_utf8( $_ )*** is necessary #to go back to utf8 encoding from the internal Perl representation #so that for the output the n-grams are in utf8 (encoded by encode_S though) $self->{allow_iproc} = 0; # allow_iproc has to be 0. Otherwise the last token in the read block will # be preprocessed and encoded in utf8, # and then attached at the beginning of the next block read from input, # which will be in the internal Perl representation } #MJ <- elsif ($params{type} eq 'byte') { $self->{skiprex} = ''; $self->{tokenrex} = ''; $self->{processtoken} = ''; } elsif ($params{type} eq 'word') { $self->{skiprex} = qr/[^a-zA-Z0-9]+/; $self->{skipinsert} = ' '; $self->{tokenrex} = qr/([a-zA-Z]+|(\d+(\.\d+)?|\d*\.\d+)([eE][-+]?\d+)?)/; $self->{processtoken} = sub { s/(\d+(\.\d+)?|\d*\.\d+)([eE][-+]?\d+)?// } } else { die "unknown type: $params{type}" } delete($params{type}); $self->{'table'} = [ ]; $self->{'total'} = [ ]; $self->{'total_distinct_count'} = 0; $self->{'lastngram'} = [ ]; $self->{'next_token_id'} = 0; $self->{'token_dict'} = { }; $self->{'token_S'} = [ ]; $self->{'token'} = [ ]; foreach my $i ( 1 .. $self->{windowsize} ) { $self->{table}[$i] = { }; $self->{total}[$i] = 0; $self->{firstngram}[$i] = ''; $self->{lastngram}[$i] = ''; } if (exists($params{'limit'})) { die "limit=$params{'limit'}" if $params{'limit'} < 1; $self->{'limit'} = $params{'limit'}; } delete($params{'limit'}); die "unknown parameters:".join(' ', %params) if %params; bless($self, $package); return $self; } sub feed_tokens { my $self = shift; # count all n-grams sizes starting from max to 1 foreach my $t1 (@_) { my $t = $t1; if (defined($self->{token_dict}->{$t})) { $t = $self->{token_dict}->{$t}; } else { my $id = $self->{next_token_id}++; $self->{token_S}->[$id] = &encode_S($t); $self->{token}->[$id] = $t; $t = $self->{token_dict}->{$t} = $id; } for (my $n=$self->{windowsize}; $n > 0; $n--) { if ($n > 1) { next unless $self->{lastngram}[$n-1] ne ''; $self->{lastngram}[$n] = $self->{lastngram}[$n-1] . ' ' . $t; } else { $self->{lastngram}[$n] = $t } if ( ($self->{table}[$n]{$self->{lastngram}[$n]} += 1)==1) { $self->{'total_distinct_count'} += 1 } $self->{'total'}[$n] += 1; if ($self->{'firstngram'}[$n] eq '') { $self->{'firstngram'}[$n] = $self->{lastngram}[$n] } } } if (exists($self->{'limit'}) and $self->{'total_distinct_count'} > 2 * $self->{'limit'}) { $self->_reduce_to_limit } } sub process_text { my $self = shift; $self->_process_text(0, @_); if (exists($self->{'limit'})) { $self->_reduce_to_limit } } sub _process_text { my $self = shift; my $cont = shift; # the minimal number of chars left for # continuation (the new-line problem, and the # problem with too long lines) # The remainder of unprocessed string is # returned. if ($cont < 0) { $cont = 0 } if ( # type is byte $self->{skiprex} eq '' and $self->{tokenrex} eq '' and $self->{processtoken} eq '' and $cont == 0 ) { return $self->_process_text_byte(@_) } my (@tokens); my $text; while (@_) { $text .= shift @_; while (length($text) > 0) { my $textl = $text; my $skip = ''; if ($self->{skiprex} ne '' && $textl =~ /^$self->{skiprex}/) { $skip = $&; $textl = $'; } if (defined($self->{skipinsert})) { $skip = $self->{skipinsert}; $text = $skip.$textl; } if (length($textl) < $cont) { last } if (length($textl)==0) { $text=$textl; last; } local $_; if ($self->{tokenrex} ne '') { if ($textl =~ /^$self->{tokenrex}/) { $_ = $&; $textl = $'; } } else { $_ = substr($textl, 0, 1); $textl = substr($textl, 1) } last if $_ eq ''; if (length($textl) < $cont) { if (defined($self->{allow_iproc}) && $self->{allow_iproc} && ref($self->{processtoken}) eq 'CODE') { &{$self->{processtoken}} } $text = $skip.$_.$textl; last; } if (ref($self->{processtoken}) eq 'CODE') { &{$self->{processtoken}} } push @tokens, $_; $text = $textl; } } $self->feed_tokens(@tokens); return $text; } sub _process_text_byte { my $self = shift; for (my $i=0; $i<=$#_; ++$i) { my @a = split('', $_[$i]); next if $#a==-1; $self->feed_tokens( @a ); } return ''; } sub process_files { my $self = shift; #MJ -> my $input_layer=''; if (defined($self->{inputlayer})) {$input_layer=$self->{inputlayer};} #MJ <- foreach my $f (@_) { my $f1; local *F; if (not ref($f)) { open(F, "$f") or die "cannot open $f:$!"; $f1 = *F } else { $f1 = $f } #MJ -> #put the encoding layer on the input when requested if ($input_layer ne '') { binmode $f1, $input_layer; } #MJ <- my ($text, $text_l, $cont) = ('', 0, 1); if ( # type is byte $self->{skiprex} eq '' and $self->{tokenrex} eq '' and $self->{processtoken} eq '' ) { $cont = 0 } while (1) { $text_l = length($text); read($f1, $text, 1024, length($text)); last if length($text) <= $text_l; $text = $self->_process_text($cont, $text); } $text = $self->_process_text(0, $text); #MJ -> #remove the encoding layer from the input stream if it was added #Caution: here is what the Perl documentation says about the pseudo layer ":pop" #"Should be considered as experimental. (...) A more elegant (and safer) interface is needed." if ($input_layer ne '') { binmode $f1,":pop"; } #MJ <- close($f1) if not ref($f); if (exists($self->{'limit'})) { $self->_reduce_to_limit } } } sub _reduce_to_limit { my $self = shift; return unless exists($self->{'limit'}) and $self->{'limit'} > 0; while ($self->{'total_distinct_count'} > $self->{'limit'}) { my $nextprunefrequency = 0; for (my $prunefrequency=1;; $prunefrequency = $nextprunefrequency) { $nextprunefrequency = $self->{'total'}[1]; foreach my $n (1 .. $self->{'windowsize'}) { my @keys = keys(%{$self->{table}[$n]}); foreach my $ngram (@keys) { my $f = $self->{table}[$n]{$ngram}; if ($f <= $prunefrequency) { delete $self->{'table'}[$n]{$ngram}; $self->{'total'}[$n] -= $prunefrequency; $self->{'total_distinct_count'} -= 1; } elsif ($nextprunefrequency > $f) { $nextprunefrequency = $f } } return if $self->{'total_distinct_count'} <= $self->{'limit'}; die if $nextprunefrequency <= $prunefrequency; } } } } # Sorts keys according to the lexicographic order. sub _keys_sorted { my $self = shift; my $n = shift; my @k = keys(%{$self->{table}[$n]}); my %k1 = (); foreach my $k (@k) { $k1{ join (' ', map { $self->{token}->[$_] } split(/ /, $k) ) } = $k; } @k = (); foreach my $k (sort(keys(%k1))) { push @k, $k1{$k}; } return @k; } sub get_ngrams { my $self = shift; my (%params) = @_; my $n = exists($params{'n'})? $params{'n'} : $self->{windowsize}; my $onlyfirst = exists($params{'onlyfirst'}) ? $params{'onlyfirst'} : ''; my $opt_normalize = exists($params{'normalize'}) ?$params{'normalize'} : ''; my $total = $self->{total}[$n]; my @keys = (); if (!exists($params{'orderby'}) or $params{'orderby'} eq 'ngram') { @keys = $self->_keys_sorted($n); } elsif ($params{'orderby'} eq 'none') { die "onlyfirst requires order" if $onlyfirst; @keys = keys(%{$self->{table}[$n]}) } elsif ($params{'orderby'} eq 'frequency') { @keys = $self->_keys_sorted($n); my %keysord = (); for (my $i=0; $i<=$#keys; ++$i) { $keysord{$keys[$i]} = $i } @keys = sort { $self->{table}[$n]{$b} <=> $self->{table}[$n]{$a} or $keysord{$a} <=> $keysord{$b} } keys(%{$self->{table}[$n]}); } else { die } @keys = splice(@keys,0,$onlyfirst) if $onlyfirst; my @ret; foreach my $ngram (@keys) { my $count = $self->{table}[$n]{$ngram}; $count = ($opt_normalize ? ($count / $total ) : $count); push @ret, $self->_encode_S($ngram), $count; } return @ret; } sub to_string { my $self = shift; my (%params) = @_; my $n = exists($params{'n'})? $params{'n'} : $self->{windowsize}; my $onlyfirst = exists($params{'onlyfirst'}) ? $params{'onlyfirst'} : ''; my $opt_normalize = exists($params{'normalize'}) ?$params{'normalize'} : ''; #my $onlyfirst = exists($params{'onlyfirst'}) ? #$params{'onlyfirst'} : ''; #delete $params{'onlyfirst'}; my $out = exists($params{'out'}) ? $params{'out'} : ''; delete $params{'out'}; my $outh = $out; if ($out and (not ref($out))) { local *FH; open(FH, ">$out") or die "cannot open $out:$!"; $outh = *FH; } #my $opt_normalize = $params{'normalize'}; #delete $params{'normalize'}; my $spartan = $params{'spartan'}; delete $params{'spartan'}; my $ret=''; $ret = "BEGIN OUTPUT BY Text::Ngrams version $VERSION\n\n" unless $spartan; foreach my $n (1 .. $self->{windowsize}) { if ($spartan and $n < $self->{windowsize}) { next } if (! $spartan ) { my $tmp = "$n-GRAMS (total count: $self->{total}[$n])"; $ret .= "$tmp\n" . "FIRST N-GRAM: ". $self->_encode_S($self->{firstngram}[$n]). "\n LAST N-GRAM: ".$self->_encode_S($self->{lastngram}[$n])."\n". ('-' x length($tmp)) . "\n"; } my $total = $self->{total}[$n]; my @keys; if (!exists($params{'orderby'}) or $params{'orderby'} eq 'ngram') { @keys = $self->_keys_sorted($n) } elsif ($params{'orderby'} eq 'none') { die "onlyfirst requires order" if $onlyfirst; @keys = keys(%{$self->{table}[$n]}) } elsif ($params{'orderby'} eq 'frequency') { @keys = sort { $self->{table}[$n]{$b} <=> $self->{table}[$n]{$a} } keys(%{$self->{table}[$n]}); } else { die } @keys = splice(@keys,0,$onlyfirst) if $onlyfirst; my %params1=%params; $params1{n}=$n; my @a = $self->get_ngrams(%params1); for (my $i=0; $i<=$#a; $i+=2) { my $ng = $a[$i]; my $f = $a[$i+1]; $ret.="$ng\t$f\n"; } if ($out) { print $outh $ret; $ret = '' } $ret .= "\n" unless $spartan; } $ret .= "END OUTPUT BY Text::Ngrams\n" unless $spartan; if ($out) { print $outh $ret; $ret = ''; close($outh) if not ref($out); } return $ret; } # http://web.cs.dal.ca/~vlado/srcperl/snip/decode_S sub decode_S ( $ ) { local $_ = shift; my $out; while (length($_) > 0) { if (/^\\(\S)/) { $_ = $'; my $tmp = $1; $tmp =~ tr/0-5Aabtnvfroil6-9NSTcEseFGRUd/\x00-\x1F\x7F/; $out .= $tmp; } elsif (/^\^_/) { $_ = $'; $out .= "\240" } elsif (/^\^(\S)/) { $_ = $'; $out .= pack('C',ord($1)+128); } elsif (/^\`(\S)/) { $_ = $'; my $tmp = $1; $tmp =~ tr/0-5Aabtnvfroil6-9NSTcEseFGRUd/\x00-\x1F\x7F/; $out .= pack('C', ord($tmp)+128); } elsif (/^_+/) { $_ = $'; my $tmp = $&; $tmp =~ tr/_/ /; $out .= $tmp; } elsif (/^[^\\^\`\s_]+/) { $_ = $'; $out .= $&; } else { die "decode_S unexpected:$_" } } return $out; } sub _encode_S { my $self = shift; my @r = (); while (@_) { push @r, map { $self->{token_S}->[$_] } split(/ /, shift @_); } return join(' ', @r); } # http://www.cs.dal.ca/~vlado/srcperl/snip/encode_S sub encode_S( $ ) { local $_ = shift; s/=/=0/g; # first hide a special character (=) s/\\/=b/g; # encode backslashes s/([\x80-\xFF])/=x$1/g; # replace >127 with 127 tr/\x80-\xFF/\x00-\x7F/; s/=x=/=X/g; # hide again = s/([\x00-\x1F\x5C\x5E-\x60\x7F])/=B$1/g; tr/\x20\x00-\x1F\x7F/_0-5Aabtnvfroil6-9NSTcEseFGRUd/; s/=x=B(\S)/`$1/g; # hex backslash s/=x(\S)/^$1/g; # hex other s/=B(\S)/\\$1/g; # backslashed s/=b/\\\\/g; # original backslashes s/=X/^=0/g; s/=0/=/g; # put back = return $_; } 1; __END__ =head1 NAME Text::Ngrams - Flexible Ngram analysis (for characters, words, and more) =head1 SYNOPSIS For default character n-gram analysis of string: use Text::Ngrams; my $ng3 = Text::Ngrams->new; $ng3->process_text('abcdefg1235678hijklmnop'); print $ng3->to_string; my @ngramsarray = $ng3->get_ngrams; One can also feed tokens manually: use Text::Ngrams; my $ng3 = Text::Ngrams->new; $ng3->feed_tokens('a'); $ng3->feed_tokens('b'); $ng3->feed_tokens('c'); $ng3->feed_tokens('d'); $ng3->feed_tokens('e'); $ng3->feed_tokens('f'); $ng3->feed_tokens('g'); $ng3->feed_tokens('h'); We can choose n-grams of various sizes, e.g.: my $ng = Text::Ngrams->new( windowsize => 6 ); or different types of n-grams, e.g.: my $ng = Text::Ngrams->new( type => byte ); my $ng = Text::Ngrams->new( type => word ); my $ng = Text::Ngrams->new( type => utf8 ); To process a list of files: $ng->process_files('somefile.txt', 'otherfile.txt'); To read the standard input or another file handle: $ng->process_files(\*STDIN); To read a file named file.txt and create a profile file file.profile of 100 most frequent, normalized byte tri-grams: use Text::Ngrams; my $ng = Text::Ngrams->new( windowsize => 3, type => byte ); $ng->process_files("file.txt"); $ng->to_string( orderby=>'frequency', onlyfirst=>100, out => "file.profile", normalize=>1, spartan=>1); =head1 DESCRIPTION This module implement text n-gram analysis, supporting several types of analysis, including character and word n-grams. The module Text::Ngrams is very flexible. For example, it allows a user to manually feed a sequence of any tokens. It handles several types of tokens (character, word), and also allows a lot of flexibility in automatic recognition and feed of tokens and the way they are combined in an n-gram. It counts all n-gram frequencies up to the maximal specified length. The output format is meant to be pretty much human-readable, while also loadable by the module. The module can be used from the command line through the script C provided with the package. =head1 OUTPUT FORMAT The output looks like this (version number may be different): BEGIN OUTPUT BY Text::Ngrams version 2.004 1-GRAMS (total count: 8) ------------------------ a 1 b 1 c 1 d 1 e 1 f 1 g 1 h 1 2-GRAMS (total count: 7) ------------------------ ab 1 bc 1 cd 1 de 1 ef 1 fg 1 gh 1 3-GRAMS (total count: 6) ------------------------ abc 1 bcd 1 cde 1 def 1 efg 1 fgh 1 END OUTPUT BY Text::Ngrams N-grams are encoded using encode_S (F), so that they can always be recognized as \S+. This encoding does not change strings "too much", e.g., letters, digits, and most punctuation characters will remail unchanged, and space is replaced by underscore (_). However, all bytes (even with code greater than 127) are encoded in unambiguous and relatively compact way. Two functions, encode_S and decode_S, are provided for translating arbitrary string into this form and vice versa. An example of word n-grams containing space: BEGIN OUTPUT BY Text::Ngrams version 2.004 1-GRAMS (total count: 8) ------------------------ The 1 brown 3 fox 3 quick 1 2-GRAMS (total count: 7) ------------------------ The_brown 1 brown_fox 2 brown_quick 1 fox_brown 2 quick_fox 1 END OUTPUT BY Text::Ngrams Or, in case of byte type of processing: BEGIN OUTPUT BY Text::Ngrams version 2.004 1-GRAMS (total count: 55) ------------------------- \t 3 \n 3 _ 12 , 2 . 3 T 1 b 3 c 1 ... etc 2-GRAMS (total count: 54) ------------------------- \t_ 1 \tT 1 \tb 1 \n\t 2 __ 5 _. 1 _b 2 _f 3 _q 1 ,\n 2 .\n 1 .. 2 Th 1 br 3 ck 1 e_ 1 ... etc END OUTPUT BY Text::Ngrams =head1 METHODS =head2 new ( windowsize => POS_INTEGER, type => 'character' | 'byte' | 'word' | 'utf8' | 'utf8_character', limit => POS_INTEGER ) my $ng = Text::Ngrams->new; my $ng = Text::Ngrams->new( windowsize=>10 ); my $ng = Text::Ngrams->new( type=>'word' ); my $ng = Text::Ngrams->new( limit=>10000 ); and similar. Creates a new C object and returns it. Parameters: =over 4 =item limit Limit the number of distinct n-grams collected during processing. Processing large files may be slow, so you can limit the total number of distinct n-grams which are counted to speed up processing. The speed-up is implemented by periodically prunning the collected n-gram. Due to this process, the final n-gram counts may not be correct, and the list of final most frequen n-grams may not be correct either. B If a limit is set, the n-gram counts at the end may not be correct due to periodical pruning of n-grams. =item windowsize n-gram size (i.e., `n' itself). Default is 3 if not given. It is stored in $object->{windowsize}. =item type Specifies a predefined type of n-grams: =over 4 =item character (default) Default character n-grams: Read letters, sequences of all other characters are replaced by a space, letters are turned uppercase. =item byte Raw character n-grams: Don't ignore any bytes and don't pre-process them. =item utf8 UTF8 characters: Variable length encoding. =item word Default word n-grams: One token is a word consisting of letters, digits and decimal digit are replaced by , and everything else is ignored. A space is inserted when n-grams are formed. =item utf8_character UTF8 analogue of the "character" type: from a UTF8 encoded text reads letters, sequences of all other characters are replaced by a space, letters are turned uppercase =back One can also modify type, creating its own type, by fine-tuning several parameters (they can be undefined): $o->{skiprex} - regular expression for ignoring stuff between tokens. $o->{skipinsert} - string to replace a skiprex match that makes string too short (efficiency issue) $o->{tokenrex} - regular expression for recognizing a token. If it is empty, it means chopping off one character. $o->{processtoken} - routine for token preprocessing. Token is given and returned in $_. $o->{allow_iproc} - boolean, if set to true (1) allows for incomplete tokens to be preprocessed and put back (efficiency motivation) $o->{inputlayer} - input layer to be put on the input stream by the function binmode before reading from a given stream and to be removed by ***binmode HANDLE,":pop"*** after the reading from the particular stream is done. Has to be a real layer (like ":encoding(utf8)"), not a pseudo layer (like ":utf8") so that the psuedo layer ":pop" is able to remove this input layer For example, the types character, byte, and word are defined in the foolowing way: if ($params{type} eq 'character') { $self->{skiprex} = ''; $self->{tokenrex} = qr/([a-zA-Z]|[^a-zA-Z]+)/; $self->{processtoken} = sub { s/[^a-zA-Z]+/ /; $_ = uc $_ } $self->{allow_iproc} = 1; } elsif ($params{type} eq 'byte') { $self->{skiprex} = ''; $self->{tokenrex} = ''; $self->{processtoken} = ''; } elsif ($params{type} eq 'utf8') { $self->{skiprex} = ''; $self->{tokenrex} = qr/([\xF0-\xF4][\x80-\xBF][\x80-\xBF][\x80-\xBF] |[\xE0-\xEF][\x80-\xBF][\x80-\xBF] |[\xC2-\xDF][\x80-\xBF] |[\x00-\xFF])/x; $self->{processtoken} = ''; } elsif ($params{type} eq 'word') { $self->{skiprex} = qr/[^a-zA-Z0-9]+/; $self->{skipinsert} = ' '; $self->{tokenrex} = qr/([a-zA-Z]+|(\d+(\.\d+)?|\d*\.\d+)([eE][-+]?\d+)?)/; $self->{processtoken} = sub { s/(\d+(\.\d+)?|\d*\.\d+)([eE][-+]?\d+)?// } } =back =head2 feed_tokens ( list of tokens ) $ng3->feed_tokens('a'); This function manually supplies tokens. =head2 process_text ( list of strings ) $ng3->process_text('abcdefg1235678hijklmnop'); $ng->process_text('The brown quick fox, brown fox, brown fox ...'); Process text, i.e., break each string into tokens and feed them. =head2 process_files ( file_names or file_handle_references) $ng->process_files('somefile.txt'); Process files, similarly to text. The files are processed line by line, so there should not be any multi-line tokens. =head2 get_ngrams ( n => NUMBER, orderby => 'ngram|frequency|none', onlyfirst => NUMBER, out => filename|handle,normalize=>1) Returns an array of requested n-grams and their friequencies in order (ngram1, f1, ngram2, f2, ...). The use of parameters is identical to the function C, except that the option 'spartan' is not applicable to C function. Parameters: =over 4 =item C The parameter C specifies the size of n-grams being retrieved. The default value is the C field. It should be less or equal than C. =back =head2 to_string ( orderby => 'ngram|frequency|none', onlyfirst => NUMBER, out => filename|handle, normalize => 1, spartan => 1 ) Some examples: print $ng3->to_string; print $ng->to_string( orderby=>'frequency' ); print $ng->to_string( orderby=>'frequency', onlyfirst=>10000 ); print $ng->to_string( orderby=>'frequency', onlyfirst=>10000, normalize=>1 ); Produce string representation of the n-gram tables. Parameters: =over 4 =item C The parameter C specifies the order of n-grams. The default value is 'ngram'. =item C The parameter C causes printing only this many first n-grams for each n. It is incompatible with C'none'>. =item C The method C produces n-gram tables. However, if those tables are large and we know that we will write them to a file right after processing, it may save memory and time to provide the parameter C, which is a filename or reference to a file handle. (Experiments on my machine do not show significant improvement nor degradation.) Filename will be opened and closed, while the file handle will not. =item C This is a boolean parameter. By default, it is false (''), in which case n-gram counts are produced. If it is true (e.g., 1), the output will contain normalized frequencies; i.e., n-gram counts divided by the total number of n-grams of the same size. =item C This is a boolean parameter. By default, it is false (''), in which case n-grams for n=1 up to the maximal value are printed. If it is true, only a list of the most frequent n-grams with the maximal length is printed. =back =head2 encode_S ( string ) This function translates any string in a /^\S*$/ compliant representation. It is primarely used in n-grams string representation to prevent white-space characters to invalidate the output format. A usage example is: $e = Text::Ngrams::encode_S( $s ); or simply $e = encode_S($s); if encode_S is imported. Encodes arbitrary string into an \S* form. See F for detailed explanation. =head2 decode_S ( string ) This is the inverse funcation of C. A usage example is: $e = Text::Ngrams::decode_S( $s ); or simply $e = decode_S($s); if decode_S is imported. Decodes a string encoded in the \S* form. See F for detailed explanation. =head1 PERFORMANCE The preformance can vary a lot depending on the type of file, in particular on the content entropy. For example a file in English is processed faster than a file in Chinese, due to a larger number of distinct n-grams. The following tests are preformed on a Pentium-III 550MHz, 512MB memory, Linux Red Hat 6 platform. (See C - the script is included in this package.) ngrams.pl --n=10 --type=byte 1Mfile The 1Mfile is a 1MB file of Chinese text. The program spent consistently 20 sec per 100KB, giving 200 seconds (3min and 20sec) for the whole file. However, after 4 minutes I gave up on waiting for n-grams to be printed. The bottleneck seems to be encode_S function, so after: ngrams.pl -n=10 --type=byte --orderby=frequency --onlyfirst=5000 1Mfile it took about 3:24 + 5 =~ 9 minutes to print. After changing C so that it provides parameter C to C in module C (see Text::Ngrams), it still took: 3:09+1:28+4:40=9.17. =head1 LIMITATIONS The method C does not handle multi-line tokens by default. This can be fixed, but it does not seem to be worth the code complication. There are various ways around this if one really needs such tokens: One way is to preprocess them. Another way is to read as much text as necessary at a time then to use C, which does handle multi-line tokens. =head1 THANKS I would like to thank cpan-testers, Jost Kriege, Shlomo Yona, David Allen (for localizing and reporting and efficiency issue with ngram prunning), Andrija, Roger Zhang, Jeremy Moses, Kevin J. Ziese, Hassen Bouzgou, Michael Ricie, and Jingyi Yang for bug reports and comments. Thanks to Chris Jordan for providing initial implementation of the function get_strings (2005). Thanks to Magdalenda Jankowska for implementing a new ngrams type utf8_character, which is very useful in processing non-English text; and for a bug fix. I will be grateful for comments, bug reports, or just letting me know that you used the module. =head1 AUTHOR Author: 2003-2013 Vlado Keselj http://web.cs.dal.ca/~vlado Contributors: 2005 Chris Jordan (contributed initial get_ngrams method) 2012 Magdalena Jankowska (utf8_character ngrams type) This module is provided "as is" without expressed or implied warranty. This is free software; you can redistribute it and/or modify it under the same terms as Perl itself. The latest version can be found at F. =head1 HISTORY This code originated in my "monkeys and rhinos" project in 2000, and is related to authorship attribution project. After our papers on authorship attribution it was reformatted as a Perl module in 2003. =head1 SEE ALSO Some of the similiar projects and related resources are the following: =over 4 =item Ngram Statistics Package in Perl, by T. Pedersen at al. This is a package that includes a script for word n-grams. =item Text::Ngram Perl Package by Simon Cozens This is another CPAN package similar to Text::Ngrams for character n-grams. As an XS implementation it is supposed to be very efficient. =item Perl script ngram.pl by Jarkko Hietaniemi This is a script for analyzing character n-grams. =item Waterloo Statistical N-Gram Language Modeling Toolkit, in C++ by Fuchun Peng A n-gram language modeling package written in C++. =item CPAN N-gram module comparison article by Ben Bullock. The page is available at F gives an interesting list of modules, although the review seem to be superficial and only partially correct. The following modules are listed in this review: Algorithm::NGram, IDS::Algorithm::Ngram, Lingua::EN::Bigram, Linuga::EN::Ngram, Lingua::Gram, Lingua::Identify, Text::Mining::Algorithm::Ngram, Text::Ngram, Text::Ngram::LanguageDetermine, Text::Ngramize, Ntext::Ngrams, and Text::Positional::Ngram. =back Some links to these resources should be available at F. =cut # $Id: Ngrams.pm 399 2013-02-17 16:01:35Z vlado $ Text-Ngrams-2.005/MANIFEST0000644000175000017500000000061712110013472014011 0ustar vladovladoChanges MANIFEST Makefile.PL Ngrams.pm README ngrams.pl t/01.out t/01.t t/02.out t/02.t t/03.out t/03.t t/04.out t/04.t t/05.in t/05.t t/05-0.in t/05-0.out t/05-1.in t/05-1.out t/05-2.in t/05-2.out t/06.t t/07.out t/07.t t/08.in t/08.out t/08.t t/09.out t/09.t t/10.out t/10.t t/11.in t/11.out t/11.t t/12.in t/12.out t/12.t t/13.out t/13.t t/14.in t/14.out t/14.t t/15.t t/15.out t/auxfunctions.pl Text-Ngrams-2.005/ngrams.pl0000755000175000017500000000765112110174225014520 0ustar vladovlado#!/usr/bin/perl -w use strict; use vars qw($VERSION); # #+ $VERSION = 2.005; #- # $Revision: 1.26 $ use Text::Ngrams; use Getopt::Long; my ($help, $version, $orderby, $onlyfirst, $limit, $spartan); my $n = 3; my $type = 'character'; sub help { print < \$n, 'normalize' => \$opt_normalize, 'type=s' => \$type, 'limit=i' => \$limit, 'help' => \$help, 'version' => \$version, 'orderby=s' => \$orderby, 'onlyfirst=i' => \$onlyfirst, 'spartan' => \$spartan); help() if $n < 1 || int($n) != $n; sub version { print $VERSION, "\n"; exit(1); } help() if $help; version() if $version; my %params = ( windowsize=>$n, type=>$type); if (defined($limit) and ($limit > 0)) { $params{'limit'} = $limit } my $ng = Text::Ngrams->new( %params ); if ($#ARGV > -1) { $ng->process_files(@ARGV) } else { $ng->process_files(\*STDIN) } %params = ( 'out' => \*STDOUT ); if (defined($orderby) and $orderby) { $params{'orderby'} = $orderby } if (defined($onlyfirst) and $onlyfirst>0) { $params{'onlyfirst'} = $onlyfirst } if ($opt_normalize) { $params{'normalize'} = $opt_normalize } if ($spartan) { $params{'spartan'} = $spartan } print $ng->to_string( %params ); exit(0); __END__ =head1 NAME ngrams - Compute the ngram frequencies and produce tables to the stdout. =head1 SYNOPIS ngram [--version] [--help] [--n=3] [--normalize] [--type=TYPE] [--orderby=ORD] [--onlyfirst=N] [input files] =head1 DESCRIPTION This script produces n-grams tables of the input files to the standard ouput. Options: =over 4 =item --normalize Prints normalized n-gram frequencies; i.e., the n-gram counts divided by the total number of n-grams of the same size. =item --onlyfirst=NUMBER Prints only the first NUMBER n-grams for each n. See Text::Ngrams module. =item --limit=NUMBER Limit the total number of distinct n-grams (for efficiency reasons, the counts may not be correct at the end). =item --version Prints version. =item --help Prints help. =item --n=NUMBER N-gram size, produces 3-grams by default. =item --orderby=frequency|ngram The n-gram order. See Text::Ngrams module. =item --type=character|byte|word|utf8 Type of n-grams produces. See Text::Ngrams module. =head1 PREREQUISITES Text::Ngrams, Getopt::Long =head1 SCRIPT CATEGORIES Text::Statistics =head1 README N-gram analysis for various kinds of n-grams (character, words, bytes, utf8, and user-defined). Based on Text::Ngrams module. =head1 SEE ALSO Text::Ngrams module. =head1 COPYRIGHT Copyright 2003-2013 Vlado Keselj F This module is provided "as is" without expressed or implied warranty. This is free software; you can redistribute it and/or modify it under the same terms as Perl itself. The latest version can be found at F. =cut Text-Ngrams-2.005/Changes0000644000175000017500000000473712110174206014165 0ustar vladovladoRevision history for Perl extension Text::Ngrams. 0.01 Wed Jun 4 20:52:49 ADT 2003 - initial version, ported from code from 2000 0.02 Fri Jun 6 16:00:00 ADT 2003 - fixed installation of ngrams.pl man page 0.03 Mon Jun 9 07:05:00 ADT 2003 - fixed test cases 3, 5, and 7 (problems found thanks to cpan-testers, Josts Smokehouse) 0.04 Jun 11 2003 - added parameter 'limit' to new - added non-ordered print - added parameter 'onlyfirst' to print 0.05 Wed Sep 3 13:00:43 ADT 2003 - fixed README 1.0 Thu Dec 18 13:09:45 AST 2003 - added parameter 'normalize' to ngrams.pl and to_string method in Ngrams.pm 1.1 Sat May 29 16:15:34 ADT 2004 - corrected typo submitted by Shlomo Yona 1.2 Thu Jul 29 15:05:28 ADT 2004 - documentation improvement - introduced first and last n-gram in the output - made _reduce_to_limit more efficient (--limit parameter) thanks to observations by David Allen 1.3 Mon Nov 1 12:52:02 AST 2004 - Improved for multi-line token processing - resolved efficiency issue with binary file (not reading line by line any more) - introduced parameters skipinsert and allow_iproc - thanks to Andrija Tomovic, Roger Zhang for suggestions 1.4 Wed Nov 3 08:45:27 AST 2004 - Fixed minor testing bugs, a warning 1.5 Wed Nov 3 14:04:30 AST 2004 - minor testing bug (rounding error) 1.6 Thu Dec 2 23:00:42 AST 2004 - added utf8 type n-grams 1.7 Wed Dec 8 11:01:07 AST 2004 - fixed a documentation typo (thanks to Kevin J. Ziese) - solving "our deprecated" warning 1.8 Thu Dec 15 11:45:38 AST 2005 - debuging - introduced spartan option in to_string - function get_ngrams added - changed output format to include space between tokens (no token separator) 1.9 Mon Nov 6 07:20:42 AST 2006 - bug fixed in get_ngrams - added option spartan to script ngrams.pl 2.001 Sat Oct 25 00:10:40 ADT 2008 - fixed bug in to_string for writing to a file with given name 2.002 Sat Oct 25 09:31:17 ADT 2008 - Improved documentation. - changed perl to $^X in test 06, which may cause problems with testing 2.003 Wed Dec 12 23:38:15 AST 2012 - minor code improvements - documentation and report improvements - testing improvement: added 14.t - introduced n-grams type utf8_character, thanks to Magdalena Jankowska - a bug fix thanks to Magdalena Jankowska 2.004 Sun Feb 17 11:28:22 AST 2013 - documentation improvements 2.005 - when sorted by frequency, introducing secondary lexicographic sorting Text-Ngrams-2.005/Makefile.PL0000644000175000017500000000233011741101406014630 0ustar vladovladouse ExtUtils::MakeMaker; my $module = 'Text::Ngrams'; my $name = 'Ngrams.pm'; (my $dir = $module) =~ s/::/-/g; WriteMakefile ( 'NAME' => $module, 'VERSION_FROM' => $name, # finds $VERSION 'dist' => { COMPRESS=>"gzip", SUFFIX=>"gz", #PREOP=>('starfish -replace -o=$dir-\$(VERSION)/README README.sfish' # #."cp -f README $dir-\$(VERSION); " # ), }, 'clean' => {FILES => "tmp* testfiles/tmp1 Text *~ Makefile.old"}, 'PREREQ_PM' => { # Foo::Ey => '1.00', }, 'PL_FILES' => {}, ($] >= 5.005 ? ## Add these new keywords supported since 5.005 (ABSTRACT_FROM => $name, # retrieve abstract from module AUTHOR => 'Vlado Keselj web.cs.dal.ca/~vlado') : () ), EXE_FILES => [ 'ngrams.pl' ], MAN3PODS => { 'Ngrams.pm' => 'blib/man3/Text::Ngrams.3' }, MAN1PODS => { 'ngrams.pl' => 'blib/man1/ngrams.pl.1' } ); open(M, ">>Makefile") or die; if ( -f 'priv.make' ) { print M getfile('priv.make') } close(M); sub getfile($) { my $f = shift; local *F; open(F, "<$f") or die "getfile:cannot open $f:$!"; my @r = ; close(F); return wantarray ? @r : join ('', @r); } Text-Ngrams-2.005/README0000644000175000017500000000202212110174225013534 0ustar vladovlado #+ Text::Ngrams version 2.005 ========================== #- This is a module for flexible N-gram analysis of a text. It supports character n-grams, word n-grams, and more. Documentation is a part of the file Ngrams.pm in the pod format. To convert it into manual page format, type something like the following: pod2html Ngrams.pm INSTALLATION To install this module type the following: perl Makefile.PL make make test make install DEPENDENCIES No significant dependecies, as far as I know. Let me know if you find something important to add. COPYRIGHT AND LICENCE Copyright 2003-2013 Vlado Keselj web.cs.dal.ca/~vlado and contributors: 2005 Chris Jordan (contributed get_ngrams) 2012 Magdalena Jankowska (utf8_character ngrams type) This script is provided "as is" without express or implied warranty. This is free software; you can redistribute it and/or modify it under the same terms as Perl itself. Text-Ngrams-2.005/t/0000700000175000017500000000000012120322302013101 5ustar vladovladoText-Ngrams-2.005/t/08.t0000644000175000017500000000156310350307337013552 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 2; use_ok("Text::Ngrams"); require 't/auxfunctions.pl'; my $ng = Text::Ngrams->new(windowsize=>2, type=>'word', limit=>4); $ng->process_files('t/08.in'); #putfile('t/08.out', $ng->to_string( orderby=>'frequency' )); my $producedout = normalize($ng->to_string( orderby=>'frequency' )); my $oldout = normalize(scalar(getfile('t/08.out'))); # ordering may vary, so let us normalize it further $producedout = &normalize1( $producedout ); $oldout = &normalize1( $oldout ); is($producedout, $oldout); sub normalize1 { my $s = shift; my $r; while ($s) { if ($s =~ /\n---+\n/) { $r .= "$`$&"; $s = $'; } else { return "$r$s" } while ($s =~ /^\S+\t(\d+)\n/) { my $n = $1; my @a; while ($s =~ /^\S+\t$n\n/) { push @a, $&; $s = $'; } $r .= join('',sort(@a)); } } return $r; } Text-Ngrams-2.005/t/09.t0000644000175000017500000000160510070324533013544 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 2; use_ok("Text::Ngrams"); require 't/auxfunctions.pl'; my $ng = Text::Ngrams->new(windowsize=>2, type=>'word'); $ng->process_files('t/05.in'); #putfile('t/09.out', $ng->to_string( orderby=>'frequency', onlyfirst=>2 )); my $producedout = normalize($ng->to_string( orderby=>'frequency', onlyfirst=>2 )); my $oldout = normalize(scalar(getfile('t/09.out'))); # ordering may vary, so let us normalize it further $producedout = &normalize1( $producedout ); $oldout = &normalize1( $oldout ); is($producedout, $oldout); sub normalize1 { my $s = shift; my $r; while ($s) { if ($s =~ /\n---+\n/) { $r .= "$`$&"; $s = $'; } else { return "$r$s" } while ($s =~ /^\S+\t(\d+)\n/) { my $n = $1; my @a; while ($s =~ /^\S+\t$n\n/) { push @a, $&; $s = $'; } $r .= join('',sort(@a)); } } return $r; } Text-Ngrams-2.005/t/05-1.in0000644000175000017500000000002610350310204014024 0ustar vladovladoM i s s i s s i p p i Text-Ngrams-2.005/t/12.out0000644000175000017500000007415212110172024014102 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 2.004 1-GRAMS (total count: 5437) FIRST N-GRAM: K LAST N-GRAM: _ --------------------------- _ 0.19477653117528 E 0.096928453191098 O 0.0706271841088836 T 0.0649255103917602 A 0.0594077616332536 I 0.0538900128747471 N 0.0535221629575133 H 0.0526025381644289 R 0.0485561890748575 S 0.0457973146956042 D 0.0340261173441236 L 0.0290601434614677 U 0.027036968916682 G 0.0231745447857274 C 0.0228066948684937 M 0.022254919992643 W 0.0213352951995586 Y 0.0200478204892404 P 0.0154496965238183 F 0.0147139966893507 B 0.0117711973514806 K 0.00772484826190914 V 0.00754092330329226 X 0.000735699834467537 Q 0.000551774875850653 J 0.000367849917233769 Z 0.000367849917233769 2-GRAMS (total count: 5436) FIRST N-GRAM: K N LAST N-GRAM: E _ --------------------------- E _ 0.0415746872700515 _ T 0.0257542310522443 T H 0.0226269315673289 D _ 0.022075055187638 T _ 0.021523178807947 H E 0.0206033848417954 S _ 0.0191317144959529 _ A 0.018027961736571 _ S 0.0165562913907285 N _ 0.0154525386313466 _ W 0.014532744665195 I N 0.014532744665195 _ I 0.0143487858719647 _ H 0.0134289919058131 O U 0.0126931567328918 Y _ 0.0115894039735099 _ C 0.0114054451802796 E R 0.0108535688005887 A N 0.010485651214128 _ B 0.0101177336276674 _ O 0.0101177336276674 O N 0.00993377483443709 H A 0.00938189845474614 R _ 0.00919793966151582 R E 0.00919793966151582 A S 0.00883002207505519 G _ 0.00864606328182487 N G 0.00864606328182487 N D 0.00846210448859456 I T 0.00827814569536424 R O 0.00827814569536424 O O 0.0079102281089036 L E 0.00772626931567329 M E 0.00772626931567329 A T 0.00754231052244297 _ M 0.00680647534952171 H I 0.00680647534952171 E D 0.00662251655629139 I S 0.00662251655629139 N E 0.00662251655629139 O _ 0.00662251655629139 _ F 0.00643855776306107 V E 0.00625459896983076 _ D 0.00607064017660044 R I 0.00607064017660044 _ N 0.00570272259013981 _ Y 0.00570272259013981 A R 0.00570272259013981 E N 0.00551876379690949 F _ 0.00551876379690949 S T 0.00551876379690949 C R 0.00533480500367918 E A 0.00515084621044886 H _ 0.00515084621044886 Y O 0.00515084621044886 _ R 0.00496688741721854 W H 0.00496688741721854 C H 0.00478292862398823 G E 0.00478292862398823 M A 0.00478292862398823 O F 0.00459896983075791 O G 0.00459896983075791 O R 0.00459896983075791 S C 0.00459896983075791 U R 0.00459896983075791 A _ 0.00441501103752759 A L 0.00441501103752759 B E 0.00441501103752759 T O 0.00441501103752759 W A 0.00441501103752759 _ G 0.00423105224429728 _ L 0.00423105224429728 _ P 0.00423105224429728 L _ 0.00423105224429728 A I 0.00404709345106696 N O 0.00404709345106696 S A 0.00404709345106696 U _ 0.00404709345106696 _ E 0.00386313465783664 _ U 0.00386313465783664 C O 0.00386313465783664 E P 0.00386313465783664 I D 0.00386313465783664 M _ 0.00386313465783664 O W 0.00386313465783664 T E 0.00386313465783664 H O 0.00367917586460633 I M 0.00367917586460633 S E 0.00367917586460633 E E 0.00349521707137601 L L 0.00349521707137601 O M 0.00349521707137601 T I 0.00349521707137601 U T 0.00349521707137601 W _ 0.00349521707137601 F O 0.0033112582781457 K E 0.0033112582781457 G O 0.00312729948491538 I _ 0.00312729948491538 N T 0.00312729948491538 3-GRAMS (total count: 5435) FIRST N-GRAM: K N O LAST N-GRAM: G E _ --------------------------- _ T H 0.0182152713891444 T H E 0.013247470101196 H E _ 0.0121435142594296 N G _ 0.00735970561177553 I N G 0.00717571297148114 A S _ 0.00699172033118675 _ A N 0.00662373505059798 N D _ 0.00643974241030359 A N D 0.00607175712971481 E D _ 0.00607175712971481 O N _ 0.00551977920883165 _ I N 0.00515179392824287 _ W H 0.00496780128794848 _ Y O 0.00496780128794848 A T _ 0.00496780128794848 Y O U 0.00496780128794848 I N _ 0.00459981600735971 M E _ 0.00459981600735971 _ A _ 0.00441582336706532 _ B E 0.00441582336706532 _ I T 0.00441582336706532 _ O F 0.00441582336706532 _ S C 0.00441582336706532 C R O 0.00441582336706532 G E _ 0.00441582336706532 R O O 0.00441582336706532 _ H A 0.00423183072677093 H A T 0.00423183072677093 O F _ 0.00423183072677093 S C R 0.00423183072677093 _ W A 0.00404783808647654 D _ S 0.00404783808647654 N _ T 0.00404783808647654 O G E 0.00404783808647654 O O G 0.00404783808647654 O U _ 0.00404783808647654 R E _ 0.00404783808647654 _ H I 0.00386384544618215 I T _ 0.00386384544618215 T O _ 0.00386384544618215 _ C O 0.00367985280588776 _ H E 0.00367985280588776 _ S A 0.00367985280588776 _ T O 0.00367985280588776 D _ T 0.00367985280588776 E _ C 0.00367985280588776 E R _ 0.00367985280588776 L E _ 0.00349586016559338 I D _ 0.00331186752529899 I S _ 0.00331186752529899 T H A 0.00331186752529899 _ G O 0.0031278748850046 _ I _ 0.0031278748850046 _ M E 0.0031278748850046 E _ T 0.0031278748850046 E _ W 0.0031278748850046 H I S 0.0031278748850046 U T _ 0.0031278748850046 V E _ 0.0031278748850046 _ C H 0.00294388224471021 _ O N 0.00294388224471021 E _ I 0.00294388224471021 E _ S 0.00294388224471021 R Y _ 0.00294388224471021 S _ T 0.00294388224471021 T _ H 0.00294388224471021 _ D O 0.00275988960441582 _ N E 0.00275988960441582 _ R E 0.00275988960441582 M A S 0.00275988960441582 A I D 0.00257589696412144 C L E 0.00257589696412144 E _ A 0.00257589696412144 I S T 0.00257589696412144 L D _ 0.00257589696412144 S _ A 0.00257589696412144 S A I 0.00257589696412144 W A S 0.00257589696412144 _ F O 0.00239190432382705 _ W I 0.00239190432382705 C H R 0.00239190432382705 D _ A 0.00239190432382705 E _ M 0.00239190432382705 E N _ 0.00239190432382705 F O R 0.00239190432382705 H O U 0.00239190432382705 H R I 0.00239190432382705 L L _ 0.00239190432382705 L Y _ 0.00239190432382705 R I S 0.00239190432382705 S T M 0.00239190432382705 T _ T 0.00239190432382705 T _ W 0.00239190432382705 T M A 0.00239190432382705 _ B U 0.00220791168353266 _ C A 0.00220791168353266 A V E 0.00220791168353266 O D _ 0.00220791168353266 O U R 0.00220791168353266 V E R 0.00220791168353266 4-GRAMS (total count: 5434) FIRST N-GRAM: K N O W LAST N-GRAM: O G E _ --------------------------- _ T H E 0.0125138019874862 T H E _ 0.00938535149061465 I N G _ 0.00662495399337505 _ A N D 0.00533676849466323 A N D _ 0.00533676849466323 _ Y O U 0.00496871549503128 _ O F _ 0.00423260949576739 _ S C R 0.00423260949576739 H A T _ 0.00423260949576739 _ I N _ 0.00404858299595142 C R O O 0.00404858299595142 O G E _ 0.00404858299595142 O O G E 0.00404858299595142 R O O G 0.00404858299595142 S C R O 0.00404858299595142 Y O U _ 0.00404858299595142 _ I T _ 0.00386455649613544 _ T O _ 0.0034965034965035 _ T H A 0.00312845049687155 H I S _ 0.00312845049687155 N _ T H 0.00294442399705558 _ H E _ 0.0027603974972396 T H A T 0.0027603974972396 _ H I S 0.00257637099742363 _ S A I 0.00257637099742363 _ W A S 0.00257637099742363 A I D _ 0.00257637099742363 D _ S C 0.00257637099742363 S A I D 0.00257637099742363 W A S _ 0.00257637099742363 _ C H R 0.00239234449760766 C H R I 0.00239234449760766 D _ T H 0.00239234449760766 E _ T H 0.00239234449760766 H R I S 0.00239234449760766 I S T M 0.00239234449760766 M A S _ 0.00239234449760766 R I S T 0.00239234449760766 S T M A 0.00239234449760766 T M A S 0.00239234449760766 H E _ C 0.00220831799779168 _ B E _ 0.00202429149797571 _ N E P 0.00202429149797571 A V E _ 0.00202429149797571 E D _ T 0.00202429149797571 E P H E 0.00202429149797571 H E W _ 0.00202429149797571 N E P H 0.00202429149797571 P H E W 0.00202429149797571 T _ T H 0.00202429149797571 _ B U T 0.00184026499815974 _ G O O 0.00184026499815974 _ H A V 0.00184026499815974 _ W I T 0.00184026499815974 B U T _ 0.00184026499815974 G O O D 0.00184026499815974 I D _ S 0.00184026499815974 O O D _ 0.00184026499815974 O U L D 0.00184026499815974 T _ H A 0.00184026499815974 W I T H 0.00184026499815974 _ M E R 0.00165623849834376 _ T I M 0.00165623849834376 _ U N C 0.00165623849834376 C L E _ 0.00165623849834376 E _ C O 0.00165623849834376 G E _ S 0.00165623849834376 I M E _ 0.00165623849834376 N C L E 0.00165623849834376 O N _ T 0.00165623849834376 O U G H 0.00165623849834376 S _ T H 0.00165623849834376 T I M E 0.00165623849834376 U L D _ 0.00165623849834376 U N C L 0.00165623849834376 _ C O U 0.00147221199852779 _ F O R 0.00147221199852779 _ N O T 0.00147221199852779 _ W H A 0.00147221199852779 A S _ T 0.00147221199852779 E _ Y O 0.00147221199852779 E R R Y 0.00147221199852779 E V E R 0.00147221199852779 F _ T H 0.00147221199852779 G H T _ 0.00147221199852779 H A V E 0.00147221199852779 I N _ T 0.00147221199852779 M E R R 0.00147221199852779 O N _ H 0.00147221199852779 O N E _ 0.00147221199852779 R R Y _ 0.00147221199852779 T H I N 0.00147221199852779 W H A T 0.00147221199852779 _ A S _ 0.00128818549871181 _ C O M 0.00128818549871181 _ D O N 0.00128818549871181 _ E V E 0.00128818549871181 _ H I M 0.00128818549871181 _ K E E 0.00128818549871181 _ O N _ 0.00128818549871181 5-GRAMS (total count: 5433) FIRST N-GRAM: K N O W _ LAST N-GRAM: O O G E _ --------------------------- _ T H E _ 0.0093870789618995 _ A N D _ 0.00533775078225658 _ S C R O 0.00404932817964292 _ Y O U _ 0.00404932817964292 C R O O G 0.00404932817964292 O O G E _ 0.00404932817964292 R O O G E 0.00404932817964292 S C R O O 0.00404932817964292 _ T H A T 0.00276090557702927 T H A T _ 0.00276090557702927 _ H I S _ 0.00257684520522731 _ S A I D 0.00257684520522731 _ W A S _ 0.00257684520522731 D _ S C R 0.00257684520522731 S A I D _ 0.00257684520522731 _ C H R I 0.00239278483342536 C H R I S 0.00239278483342536 H R I S T 0.00239278483342536 I S T M A 0.00239278483342536 N _ T H E 0.00239278483342536 R I S T M 0.00239278483342536 S T M A S 0.00239278483342536 T M A S _ 0.00239278483342536 _ N E P H 0.00202466408982146 E P H E W 0.00202466408982146 N E P H E 0.00202466408982146 P H E W _ 0.00202466408982146 _ B U T _ 0.00184060371801951 _ G O O D 0.00184060371801951 _ W I T H 0.00184060371801951 D _ T H E 0.00184060371801951 G O O D _ 0.00184060371801951 I D _ S C 0.00184060371801951 T H E _ C 0.00184060371801951 _ T I M E 0.00165654334621756 _ U N C L 0.00165654334621756 A I D _ S 0.00165654334621756 N C L E _ 0.00165654334621756 O U L D _ 0.00165654334621756 T I M E _ 0.00165654334621756 U N C L E 0.00165654334621756 _ H A V E 0.00147248297441561 _ I N _ T 0.00147248297441561 _ M E R R 0.00147248297441561 _ W H A T 0.00147248297441561 E _ Y O U 0.00147248297441561 E D _ T H 0.00147248297441561 E R R Y _ 0.00147248297441561 H A V E _ 0.00147248297441561 I N _ T H 0.00147248297441561 M E R R Y 0.00147248297441561 O G E _ S 0.00147248297441561 W H A T _ 0.00147248297441561 _ F O R _ 0.00128842260261366 _ K E E P 0.00128842260261366 _ O F _ T 0.00128842260261366 _ U P O N 0.00128842260261366 E _ T H E 0.00128842260261366 F _ T H E 0.00128842260261366 K E E P _ 0.00128842260261366 O U G H _ 0.00128842260261366 R Y _ C H 0.00128842260261366 T _ T H E 0.00128842260261366 T H I N G 0.00128842260261366 U P O N _ 0.00128842260261366 W I T H _ 0.00128842260261366 _ E V E R 0.00110436223081171 _ N O T _ 0.00110436223081171 _ T H E I 0.00110436223081171 A S _ T H 0.00110436223081171 G _ T H E 0.00110436223081171 G E _ S _ 0.00110436223081171 H E I R _ 0.00110436223081171 H I N G _ 0.00110436223081171 I G H T _ 0.00110436223081171 I N G _ O 0.00110436223081171 O F _ T H 0.00110436223081171 R R Y _ C 0.00110436223081171 S _ N E P 0.00110436223081171 T _ H E _ 0.00110436223081171 T H E _ N 0.00110436223081171 T H E I R 0.00110436223081171 V E R Y _ 0.00110436223081171 Y _ C H R 0.00110436223081171 _ A L L _ 0.000920301859009755 _ C L E R 0.000920301859009755 _ D O N _ 0.000920301859009755 _ F R O M 0.000920301859009755 _ H A D _ 0.000920301859009755 _ H I M _ 0.000920301859009755 _ I T _ H 0.000920301859009755 _ I T _ W 0.000920301859009755 _ W H Y _ 0.000920301859009755 _ W O U L 0.000920301859009755 _ Y O U R 0.000920301859009755 A N D _ W 0.000920301859009755 A V E _ Y 0.000920301859009755 C L E R K 0.000920301859009755 D O N _ T 0.000920301859009755 E _ A N D 0.000920301859009755 6-GRAMS (total count: 5432) FIRST N-GRAM: K N O W _ H LAST N-GRAM: R O O G E _ --------------------------- _ S C R O O 0.0040500736377025 C R O O G E 0.0040500736377025 R O O G E _ 0.0040500736377025 S C R O O G 0.0040500736377025 _ T H A T _ 0.00276141384388807 _ S A I D _ 0.00257731958762887 D _ S C R O 0.00257731958762887 _ C H R I S 0.00239322533136966 C H R I S T 0.00239322533136966 H R I S T M 0.00239322533136966 I S T M A S 0.00239322533136966 R I S T M A 0.00239322533136966 S T M A S _ 0.00239322533136966 _ N E P H E 0.00202503681885125 E P H E W _ 0.00202503681885125 N E P H E W 0.00202503681885125 _ G O O D _ 0.00184094256259205 _ T H E _ C 0.00184094256259205 I D _ S C R 0.00184094256259205 N _ T H E _ 0.00184094256259205 _ T I M E _ 0.00165684830633284 _ U N C L E 0.00165684830633284 A I D _ S C 0.00165684830633284 S A I D _ S 0.00165684830633284 U N C L E _ 0.00165684830633284 _ H A V E _ 0.00147275405007364 _ I N _ T H 0.00147275405007364 _ M E R R Y 0.00147275405007364 _ W H A T _ 0.00147275405007364 D _ T H E _ 0.00147275405007364 E _ Y O U _ 0.00147275405007364 M E R R Y _ 0.00147275405007364 O O G E _ S 0.00147275405007364 _ K E E P _ 0.00128865979381443 _ U P O N _ 0.00128865979381443 _ W I T H _ 0.00128865979381443 E D _ T H E 0.00128865979381443 I N _ T H E 0.00128865979381443 T _ T H E _ 0.00128865979381443 _ O F _ T H 0.00110456553755523 _ T H E _ N 0.00110456553755523 _ T H E I R 0.00110456553755523 E R R Y _ C 0.00110456553755523 F _ T H E _ 0.00110456553755523 O F _ T H E 0.00110456553755523 O G E _ S _ 0.00110456553755523 R R Y _ C H 0.00110456553755523 R Y _ C H R 0.00110456553755523 S _ N E P H 0.00110456553755523 T H E I R _ 0.00110456553755523 T H I N G _ 0.00110456553755523 Y _ C H R I 0.00110456553755523 _ A N D _ W 0.000920471281296024 _ C L E R K 0.000920471281296024 _ D O N _ T 0.000920471281296024 _ F R O M _ 0.000920471281296024 _ I T _ H A 0.000920471281296024 _ W O U L D 0.000920471281296024 _ Y O U _ T 0.000920471281296024 A V E _ Y O 0.000920471281296024 C L E R K _ 0.000920471281296024 D O N _ T _ 0.000920471281296024 E _ A N D _ 0.000920471281296024 E _ T H E _ 0.000920471281296024 E _ W A S _ 0.000920471281296024 G O O D _ A 0.000920471281296024 N _ H I S _ 0.000920471281296024 S _ A N D _ 0.000920471281296024 T _ W A S _ 0.000920471281296024 T H A T _ H 0.000920471281296024 T H E _ N E 0.000920471281296024 T H O U G H 0.000920471281296024 V E _ Y O U 0.000920471281296024 W O U L D _ 0.000920471281296024 Y _ A N D _ 0.000920471281296024 _ A _ T I M 0.000736377025036819 _ A N D _ H 0.000736377025036819 _ A N D _ T 0.000736377025036819 _ C A M E _ 0.000736377025036819 _ C O M E _ 0.000736377025036819 _ E V E R Y 0.000736377025036819 _ I T _ W A 0.000736377025036819 _ R E T U R 0.000736377025036819 _ S _ N E P 0.000736377025036819 _ T H E _ F 0.000736377025036819 _ T H O U G 0.000736377025036819 _ T O _ B E 0.000736377025036819 _ W E R E _ 0.000736377025036819 _ W H E N _ 0.000736377025036819 _ Y O U R _ 0.000736377025036819 A _ T I M E 0.000736377025036819 A S _ T H E 0.000736377025036819 A T _ H E _ 0.000736377025036819 A T I O N _ 0.000736377025036819 E _ C L E R 0.000736377025036819 E _ N E P H 0.000736377025036819 E _ S _ N E 0.000736377025036819 E E P _ I T 0.000736377025036819 E T U R N E 0.000736377025036819 G _ T H E I 0.000736377025036819 7-GRAMS (total count: 5431) FIRST N-GRAM: K N O W _ H I LAST N-GRAM: C R O O G E _ --------------------------- _ S C R O O G 0.00405081937028172 C R O O G E _ 0.00405081937028172 S C R O O G E 0.00405081937028172 D _ S C R O O 0.00257779414472473 _ C H R I S T 0.00239366599153011 C H R I S T M 0.00239366599153011 H R I S T M A 0.00239366599153011 I S T M A S _ 0.00239366599153011 R I S T M A S 0.00239366599153011 _ N E P H E W 0.00202540968514086 N E P H E W _ 0.00202540968514086 I D _ S C R O 0.00184128153194623 _ S A I D _ S 0.00165715337875161 _ U N C L E _ 0.00165715337875161 A I D _ S C R 0.00165715337875161 S A I D _ S C 0.00165715337875161 _ M E R R Y _ 0.00147302522555699 R O O G E _ S 0.00147302522555699 _ I N _ T H E 0.00128889707236236 E D _ T H E _ 0.00128889707236236 I N _ T H E _ 0.00128889707236236 _ O F _ T H E 0.00110476891916774 _ T H E I R _ 0.00110476891916774 E R R Y _ C H 0.00110476891916774 M E R R Y _ C 0.00110476891916774 O F _ T H E _ 0.00110476891916774 O O G E _ S _ 0.00110476891916774 R R Y _ C H R 0.00110476891916774 R Y _ C H R I 0.00110476891916774 S _ N E P H E 0.00110476891916774 Y _ C H R I S 0.00110476891916774 _ C L E R K _ 0.000920640765973117 _ D O N _ T _ 0.000920640765973117 _ G O O D _ A 0.000920640765973117 _ T H A T _ H 0.000920640765973117 _ T H E _ N E 0.000920640765973117 _ W O U L D _ 0.000920640765973117 A V E _ Y O U 0.000920640765973117 E _ Y O U _ T 0.000920640765973117 V E _ Y O U _ 0.000920640765973117 _ A _ T I M E 0.000736512612778494 _ H A V E _ Y 0.000736512612778494 _ I T _ H A S 0.000736512612778494 _ I T _ W A S 0.000736512612778494 _ K E E P _ I 0.000736512612778494 _ R E T U R N 0.000736512612778494 _ S _ N E P H 0.000736512612778494 _ T H A T _ I 0.000736512612778494 _ T H E _ C L 0.000736512612778494 _ T H O U G H 0.000736512612778494 _ T O _ B E _ 0.000736512612778494 _ W H A T _ R 0.000736512612778494 _ Y O U _ T O 0.000736512612778494 A _ T I M E _ 0.000736512612778494 A S _ T H E _ 0.000736512612778494 D _ T H E _ N 0.000736512612778494 E _ C L E R K 0.000736512612778494 E _ N E P H E 0.000736512612778494 E _ S _ N E P 0.000736512612778494 E T U R N E D 0.000736512612778494 G _ T H E I R 0.000736512612778494 G E _ S _ N E 0.000736512612778494 G H T _ H A V 0.000736512612778494 H A T _ H E _ 0.000736512612778494 H A V E _ Y O 0.000736512612778494 H E _ C L E R 0.000736512612778494 H E _ N E P H 0.000736512612778494 H T _ H A V E 0.000736512612778494 I G H T _ H A 0.000736512612778494 I N G _ I N _ 0.000736512612778494 I T _ H A S _ 0.000736512612778494 I T _ W A S _ 0.000736512612778494 K E E P _ I T 0.000736512612778494 N E D _ T H E 0.000736512612778494 O G E _ S _ N 0.000736512612778494 O U _ T O _ B 0.000736512612778494 R E T U R N E 0.000736512612778494 R N E D _ T H 0.000736512612778494 T _ H A V E _ 0.000736512612778494 T _ T H E _ C 0.000736512612778494 T H A T _ H E 0.000736512612778494 T H E _ C L E 0.000736512612778494 T H E _ N E P 0.000736512612778494 T U R N E D _ 0.000736512612778494 U _ T O _ B E 0.000736512612778494 U R N E D _ T 0.000736512612778494 Y O U _ T O _ 0.000736512612778494 _ A F T E R N 0.00055238445958387 _ E V E R Y _ 0.00055238445958387 _ H U M B U G 0.00055238445958387 _ L E T _ M E 0.00055238445958387 _ M I G H T _ 0.00055238445958387 _ O F _ S C R 0.00055238445958387 _ R E A S O N 0.00055238445958387 _ S A I D _ T 0.00055238445958387 _ T H E _ C O 0.00055238445958387 _ T I M E _ F 0.00055238445958387 _ U P O N _ T 0.00055238445958387 _ W A S _ T H 0.00055238445958387 _ W H I C H _ 0.00055238445958387 8-GRAMS (total count: 5430) FIRST N-GRAM: K N O W _ H I M LAST N-GRAM: S C R O O G E _ --------------------------- _ S C R O O G E 0.00405156537753223 S C R O O G E _ 0.00405156537753223 D _ S C R O O G 0.00257826887661142 _ C H R I S T M 0.00239410681399632 C H R I S T M A 0.00239410681399632 H R I S T M A S 0.00239410681399632 R I S T M A S _ 0.00239410681399632 _ N E P H E W _ 0.00202578268876611 I D _ S C R O O 0.00184162062615101 _ S A I D _ S C 0.00165745856353591 A I D _ S C R O 0.00165745856353591 S A I D _ S C R 0.00165745856353591 C R O O G E _ S 0.00147329650092081 _ I N _ T H E _ 0.00128913443830571 _ M E R R Y _ C 0.00110497237569061 _ O F _ T H E _ 0.00110497237569061 E R R Y _ C H R 0.00110497237569061 M E R R Y _ C H 0.00110497237569061 R O O G E _ S _ 0.00110497237569061 R R Y _ C H R I 0.00110497237569061 R Y _ C H R I S 0.00110497237569061 S _ N E P H E W 0.00110497237569061 Y _ C H R I S T 0.00110497237569061 A V E _ Y O U _ 0.000920810313075506 _ A _ T I M E _ 0.000736648250460405 _ H A V E _ Y O 0.000736648250460405 _ I T _ H A S _ 0.000736648250460405 _ I T _ W A S _ 0.000736648250460405 _ K E E P _ I T 0.000736648250460405 _ R E T U R N E 0.000736648250460405 _ S _ N E P H E 0.000736648250460405 _ T H A T _ H E 0.000736648250460405 _ T H E _ C L E 0.000736648250460405 _ T H E _ N E P 0.000736648250460405 _ Y O U _ T O _ 0.000736648250460405 D _ T H E _ N E 0.000736648250460405 E _ C L E R K _ 0.000736648250460405 E _ N E P H E W 0.000736648250460405 E _ S _ N E P H 0.000736648250460405 E _ Y O U _ T O 0.000736648250460405 E T U R N E D _ 0.000736648250460405 G _ T H E I R _ 0.000736648250460405 G E _ S _ N E P 0.000736648250460405 G H T _ H A V E 0.000736648250460405 H A V E _ Y O U 0.000736648250460405 H E _ C L E R K 0.000736648250460405 H E _ N E P H E 0.000736648250460405 H T _ H A V E _ 0.000736648250460405 I G H T _ H A V 0.000736648250460405 N E D _ T H E _ 0.000736648250460405 O G E _ S _ N E 0.000736648250460405 O O G E _ S _ N 0.000736648250460405 O U _ T O _ B E 0.000736648250460405 R E T U R N E D 0.000736648250460405 R N E D _ T H E 0.000736648250460405 T H A T _ H E _ 0.000736648250460405 T H E _ C L E R 0.000736648250460405 T H E _ N E P H 0.000736648250460405 T U R N E D _ T 0.000736648250460405 U _ T O _ B E _ 0.000736648250460405 U R N E D _ T H 0.000736648250460405 V E _ Y O U _ T 0.000736648250460405 Y O U _ T O _ B 0.000736648250460405 _ A F T E R N O 0.000552486187845304 _ G O O D _ A F 0.000552486187845304 _ H U M B U G _ 0.000552486187845304 _ L E T _ M E _ 0.000552486187845304 _ O F _ S C R O 0.000552486187845304 _ R E A S O N _ 0.000552486187845304 _ S A I D _ T H 0.000552486187845304 _ T H A T _ I T 0.000552486187845304 _ T I M E _ F O 0.000552486187845304 _ T O _ B E _ M 0.000552486187845304 _ U N C L E _ S 0.000552486187845304 _ U P O N _ T H 0.000552486187845304 _ W A S _ T H E 0.000552486187845304 _ Y O U _ D O N 0.000552486187845304 _ Y O U _ R E _ 0.000552486187845304 A _ T I M E _ F 0.000552486187845304 A F T E R N O O 0.000552486187845304 A S _ W H A T _ 0.000552486187845304 C R O O G E _ H 0.000552486187845304 D _ A F T E R N 0.000552486187845304 E _ I T _ W A S 0.000552486187845304 E _ M E R R Y _ 0.000552486187845304 E _ U N C L E _ 0.000552486187845304 E D _ S C R O O 0.000552486187845304 E D _ T H E _ N 0.000552486187845304 E P H E W _ W H 0.000552486187845304 F _ S C R O O G 0.000552486187845304 F T E R N O O N 0.000552486187845304 G _ I N _ T H E 0.000552486187845304 G O O D _ A F T 0.000552486187845304 H A T _ H E _ W 0.000552486187845304 I M E _ F O R _ 0.000552486187845304 I N G _ I N _ T 0.000552486187845304 I N G _ Y O U R 0.000552486187845304 K E E P _ I T _ 0.000552486187845304 N _ S A I D _ S 0.000552486187845304 N E P H E W _ W 0.000552486187845304 9-GRAMS (total count: 5429) FIRST N-GRAM: K N O W _ H I M _ LAST N-GRAM: _ S C R O O G E _ --------------------------- _ S C R O O G E _ 0.00405231165960582 D _ S C R O O G E 0.00257874378338552 _ C H R I S T M A 0.00239454779885798 C H R I S T M A S 0.00239454779885798 H R I S T M A S _ 0.00239454779885798 I D _ S C R O O G 0.00184195984527537 _ S A I D _ S C R 0.00165776386074784 A I D _ S C R O O 0.00165776386074784 S A I D _ S C R O 0.00165776386074784 S C R O O G E _ S 0.0014735678762203 _ M E R R Y _ C H 0.00110517590716522 C R O O G E _ S _ 0.00110517590716522 E R R Y _ C H R I 0.00110517590716522 M E R R Y _ C H R 0.00110517590716522 R R Y _ C H R I S 0.00110517590716522 R Y _ C H R I S T 0.00110517590716522 S _ N E P H E W _ 0.00110517590716522 Y _ C H R I S T M 0.00110517590716522 _ H A V E _ Y O U 0.000736783938110149 _ R E T U R N E D 0.000736783938110149 _ S _ N E P H E W 0.000736783938110149 _ T H A T _ H E _ 0.000736783938110149 _ T H E _ C L E R 0.000736783938110149 _ T H E _ N E P H 0.000736783938110149 _ Y O U _ T O _ B 0.000736783938110149 A V E _ Y O U _ T 0.000736783938110149 D _ T H E _ N E P 0.000736783938110149 E _ N E P H E W _ 0.000736783938110149 E _ S _ N E P H E 0.000736783938110149 E _ Y O U _ T O _ 0.000736783938110149 E T U R N E D _ T 0.000736783938110149 G E _ S _ N E P H 0.000736783938110149 G H T _ H A V E _ 0.000736783938110149 H A V E _ Y O U _ 0.000736783938110149 H E _ C L E R K _ 0.000736783938110149 H E _ N E P H E W 0.000736783938110149 I G H T _ H A V E 0.000736783938110149 O G E _ S _ N E P 0.000736783938110149 O O G E _ S _ N E 0.000736783938110149 O U _ T O _ B E _ 0.000736783938110149 R E T U R N E D _ 0.000736783938110149 R N E D _ T H E _ 0.000736783938110149 R O O G E _ S _ N 0.000736783938110149 T H E _ C L E R K 0.000736783938110149 T H E _ N E P H E 0.000736783938110149 T U R N E D _ T H 0.000736783938110149 U R N E D _ T H E 0.000736783938110149 V E _ Y O U _ T O 0.000736783938110149 Y O U _ T O _ B E 0.000736783938110149 _ A _ T I M E _ F 0.000552587953582612 _ A F T E R N O O 0.000552587953582612 _ G O O D _ A F T 0.000552587953582612 _ K E E P _ I T _ 0.000552587953582612 _ N E P H E W _ W 0.000552587953582612 _ O F _ S C R O O 0.000552587953582612 _ T H A T _ I T _ 0.000552587953582612 _ T I M E _ F O R 0.000552587953582612 _ U P O N _ T H E 0.000552587953582612 _ W A S _ T H E _ 0.000552587953582612 _ Y O U _ D O N _ 0.000552587953582612 A _ T I M E _ F O 0.000552587953582612 A F T E R N O O N 0.000552587953582612 D _ A F T E R N O 0.000552587953582612 E _ I T _ W A S _ 0.000552587953582612 E D _ S C R O O G 0.000552587953582612 E D _ T H E _ N E 0.000552587953582612 F _ S C R O O G E 0.000552587953582612 F T E R N O O N _ 0.000552587953582612 G _ I N _ T H E _ 0.000552587953582612 G O O D _ A F T E 0.000552587953582612 I N G _ I N _ T H 0.000552587953582612 N _ S A I D _ S C 0.000552587953582612 N E P H E W _ W H 0.000552587953582612 N G _ I N _ T H E 0.000552587953582612 O D _ A F T E R N 0.000552587953582612 O F _ S C R O O G 0.000552587953582612 O O D _ A F T E R 0.000552587953582612 O U _ D O N _ T _ 0.000552587953582612 S C R O O G E _ H 0.000552587953582612 T H A T _ H E _ W 0.000552587953582612 T I M E _ F O R _ 0.000552587953582612 U _ T O _ B E _ M 0.000552587953582612 Y O U _ D O N _ T 0.000552587953582612 _ A _ M E R R Y _ 0.000368391969055075 _ A M _ S U R E _ 0.000368391969055075 _ A N D _ W A S _ 0.000368391969055075 _ A N O T H E R _ 0.000368391969055075 _ A P A R T _ F R 0.000368391969055075 _ A S _ I F _ T H 0.000368391969055075 _ B E _ M E R R Y 0.000368391969055075 _ B E C A U S E _ 0.000368391969055075 _ B U T _ Y O U _ 0.000368391969055075 _ C O U N T I N G 0.000368391969055075 _ D O N _ T _ B E 0.000368391969055075 _ F E L L _ I N _ 0.000368391969055075 _ F R O M _ Y O U 0.000368391969055075 _ G O O D _ A N D 0.000368391969055075 _ H E _ C O U L D 0.000368391969055075 _ H E _ W O U L D 0.000368391969055075 _ H I M S E L F _ 0.000368391969055075 10-GRAMS (total count: 5428) FIRST N-GRAM: K N O W _ H I M _ A LAST N-GRAM: D _ S C R O O G E _ ---------------------------- D _ S C R O O G E _ 0.0025792188651437 _ C H R I S T M A S 0.00239498894620486 C H R I S T M A S _ 0.00239498894620486 I D _ S C R O O G E 0.00184229918938836 _ S A I D _ S C R O 0.00165806927044952 A I D _ S C R O O G 0.00165806927044952 S A I D _ S C R O O 0.00165806927044952 _ S C R O O G E _ S 0.00147383935151069 _ M E R R Y _ C H R 0.00110537951363301 E R R Y _ C H R I S 0.00110537951363301 M E R R Y _ C H R I 0.00110537951363301 R R Y _ C H R I S T 0.00110537951363301 R Y _ C H R I S T M 0.00110537951363301 S C R O O G E _ S _ 0.00110537951363301 Y _ C H R I S T M A 0.00110537951363301 _ H A V E _ Y O U _ 0.000736919675755343 _ R E T U R N E D _ 0.000736919675755343 _ S _ N E P H E W _ 0.000736919675755343 _ T H E _ C L E R K 0.000736919675755343 _ T H E _ N E P H E 0.000736919675755343 _ Y O U _ T O _ B E 0.000736919675755343 A V E _ Y O U _ T O 0.000736919675755343 C R O O G E _ S _ N 0.000736919675755343 D _ T H E _ N E P H 0.000736919675755343 E _ S _ N E P H E W 0.000736919675755343 E _ Y O U _ T O _ B 0.000736919675755343 E T U R N E D _ T H 0.000736919675755343 G E _ S _ N E P H E 0.000736919675755343 H A V E _ Y O U _ T 0.000736919675755343 H E _ N E P H E W _ 0.000736919675755343 I G H T _ H A V E _ 0.000736919675755343 O G E _ S _ N E P H 0.000736919675755343 O O G E _ S _ N E P 0.000736919675755343 R E T U R N E D _ T 0.000736919675755343 R O O G E _ S _ N E 0.000736919675755343 T H E _ C L E R K _ 0.000736919675755343 T H E _ N E P H E W 0.000736919675755343 T U R N E D _ T H E 0.000736919675755343 U R N E D _ T H E _ 0.000736919675755343 V E _ Y O U _ T O _ 0.000736919675755343 Y O U _ T O _ B E _ 0.000736919675755343 _ A _ T I M E _ F O 0.000552689756816507 _ A F T E R N O O N 0.000552689756816507 _ G O O D _ A F T E 0.000552689756816507 _ N E P H E W _ W H 0.000552689756816507 _ O F _ S C R O O G 0.000552689756816507 _ S C R O O G E _ H 0.000552689756816507 _ T H A T _ H E _ W 0.000552689756816507 _ T I M E _ F O R _ 0.000552689756816507 _ Y O U _ D O N _ T 0.000552689756816507 A _ T I M E _ F O R 0.000552689756816507 A F T E R N O O N _ 0.000552689756816507 D _ A F T E R N O O 0.000552689756816507 E D _ S C R O O G E 0.000552689756816507 E D _ T H E _ N E P 0.000552689756816507 F _ S C R O O G E _ 0.000552689756816507 G O O D _ A F T E R 0.000552689756816507 I N G _ I N _ T H E 0.000552689756816507 N _ S A I D _ S C R 0.000552689756816507 N G _ I N _ T H E _ 0.000552689756816507 O D _ A F T E R N O 0.000552689756816507 O F _ S C R O O G E 0.000552689756816507 O O D _ A F T E R N 0.000552689756816507 O U _ T O _ B E _ M 0.000552689756816507 Y O U _ D O N _ T _ 0.000552689756816507 _ A _ M E R R Y _ C 0.000368459837877671 _ A M _ S U R E _ I 0.000368459837877671 _ A P A R T _ F R O 0.000368459837877671 _ B E _ M E R R Y _ 0.000368459837877671 _ C O U N T I N G _ 0.000368459837877671 _ D O N _ T _ B E _ 0.000368459837877671 _ F E L L _ I N _ L 0.000368459837877671 _ F R O M _ Y O U _ 0.000368459837877671 _ G O O D _ A N D _ 0.000368459837877671 _ H E _ W O U L D _ 0.000368459837877671 _ I _ A M _ S U R E 0.000368459837877671 _ I T _ W A S _ T H 0.000368459837877671 _ M E _ G O O D _ A 0.000368459837877671 _ M I G H T _ H A V 0.000368459837877671 _ M U C H _ G O O D 0.000368459837877671 _ R E A S O N _ H A 0.000368459837877671 _ R I G H T _ H A V 0.000368459837877671 _ S A I D _ T H A T 0.000368459837877671 _ S C R O O G E _ A 0.000368459837877671 _ S C R O O G E _ I 0.000368459837877671 _ S C R O O G E _ M 0.000368459837877671 _ T H E _ C O U R T 0.000368459837877671 _ T H E _ O N L Y _ 0.000368459837877671 _ T H E _ U N C L E 0.000368459837877671 _ T H E _ Y E A R _ 0.000368459837877671 _ T O _ B E _ M E R 0.000368459837877671 _ U N C L E _ S A I 0.000368459837877671 _ U P O N _ T H E _ 0.000368459837877671 _ W A S _ T H E _ V 0.000368459837877671 _ W H A T _ R E A S 0.000368459837877671 _ W H A T _ R I G H 0.000368459837877671 _ W O U L D _ S E E 0.000368459837877671 A _ M E R R Y _ C H 0.000368459837877671 A I D _ T H A T _ H 0.000368459837877671 A M _ S U R E _ I _ 0.000368459837877671 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/09.out0000644000175000017500000000044510350307337014115 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 1.8 1-GRAMS (total count: 8) FIRST N-GRAM: The LAST N-GRAM: fox ------------------------ brown 3 fox 3 2-GRAMS (total count: 7) FIRST N-GRAM: The brown LAST N-GRAM: brown fox ------------------------ brown fox 2 fox brown 2 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/15.t0000644000175000017500000000053012062250003013526 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 2; BEGIN { use_ok("Text::Ngrams", qw(encode_S decode_S)) } require 't/auxfunctions.pl'; use strict; my $ng = Text::Ngrams->new( type => "byte", windowsize=>5); $ng->process_text('abc'); #$ng->to_string(spartan=>1, 'out' => 't/15.out'); isn('t/15.out',$ng->to_string(spartan=>1, 'out' => 't/15.out')); Text-Ngrams-2.005/t/12.in0000644000175000017500000001341310141466011013676 0ustar vladovladoknow him; and when they saw him coming on, would tug their owners into doorways and up courts; and then would wag their tails as though they said, `No eye at all is better than an evil eye, dark master!' But what did Scrooge care! It was the very thing he liked. To edge his way along the crowded paths of life, warning all human sympathy to keep its distance, was what the knowing ones call `nuts' to Scrooge. Once upon a time -- of all the good days in the year, on Christmas Eve -- old Scrooge sat busy in his counting-house. It was cold, bleak, biting weather: foggy withal: and he could hear the people in the court outside, go wheezing up and down, beating their hands upon their breasts, and stamping their feet upon the pavement stones to warm them. The city clocks had only just gone three, but it was quite dark already -- it had not been light all day -- and candles were flaring in the windows of the neighbouring offices, like ruddy smears upon the palpable brown air. The fog came pouring in at every chink and keyhole, and was so dense without, that although the court was of the narrowest, the houses opposite were mere phantoms. To see the dingy cloud come drooping down, obscuring everything, one might have thought that Nature lived hard by, and was brewing on a large scale. The door of Scrooge's counting-house was open that he might keep his eye upon his clerk, who in a dismal little cell beyond, a sort of tank, was copying letters. Scrooge had a very small fire, but the clerk's fire was so very much smaller that it looked like one coal. But he couldn't replenish it, for Scrooge kept the coal-box in his own room; and so surely as the clerk came in with the shovel, the master predicted that it would be necessary for them to part. Wherefore the clerk put on his white comforter, and tried to warm himself at the candle; in which effort, not being a man of a strong imagination, he failed. `A merry Christmas, uncle! God save you!' cried a cheerful voice. It was the voice of Scrooge's nephew, who came upon him so quickly that this was the first intimation he had of his approach. `Bah!' said Scrooge, `Humbug!' He had so heated himself with rapid walking in the fog and frost, this nephew of Scrooge's, that he was all in a glow; his face was ruddy and handsome; his eyes sparkled, and his breath smoked again. `Christmas a humbug, uncle!' said Scrooge's nephew. `You don't mean that, I am sure?' `I do,' said Scrooge. `Merry Christmas! What right have you to be merry? What reason have you to be merry? You're poor enough.' `Come, then,' returned the nephew gaily. `What right have you to be dismal? What reason have you to be morose? You're rich enough.' Scrooge having no better answer ready on the spur of the moment, said `Bah!' again; and followed it up with `Humbug.' `Don't be cross, uncle!' said the nephew. `What else can I be,' returned the uncle, `when I live in such a world of fools as this? Merry Christmas! Out upon merry Christmas! What's Christmas time to you but a time for paying bills without money; a time for finding yourself a year older, but not an hour richer; a time for balancing your books and having every item in `em through a round dozen of months presented dead against you? If I could work my will,' said Scrooge indignantly, `every idiot who goes about with "Merry Christmas" on his lips, should be boiled with his own pudding, and buried with a stake of holly through his heart. He should!' `Uncle!' pleaded the nephew. `Nephew!' returned the uncle sternly, `keep Christmas in your own way, and let me keep it in mine.' `Keep it!' repeated Scrooge's nephew. `But you don't keep it.' `Let me leave it alone, then,' said Scrooge. `Much good may it do you! Much good it has ever done you!' `There are many things from which I might have derived good, by which I have not profited, I dare say,' returned the nephew. `Christmas among the rest. But I am sure I have always thought of Christmas time, when it has come round -- apart from the veneration due to its sacred name and origin, if anything belonging to it can be apart from that -- as a good time; a kind, forgiving, charitable, pleasant time: the only time I know of, in the long calendar of the year, when men and women seem by one consent to open their shut-up hearts freely, and to think of people below them as if they really were fellow-passengers to the grave, and not another race of creatures bound on other journeys. And therefore, uncle, though it has never put a scrap of gold or silver in my pocket, I believe that it has done me good, and will do me good; and I say, God bless it!' The clerk in the Tank involuntarily applauded. Becoming immediately sensible of the impropriety, he poked the fire, and extinguished the last frail spark for ever. `Let me hear another sound from you,' said Scrooge, `and you'll keep your Christmas by losing your situation! You're quite a powerful speaker, sir,' he added, turning to his nephew. `I wonder you don't go into Parliament.' `Don't be angry, uncle. Come! Dine with us tomorrow.' Scrooge said that he would see him -- yes, indeed he did. He went the whole length of the expression, and said that he would see him in that extremity first. `But why?' cried Scrooge's nephew. `Why?' `Why did you get married?' said Scrooge. `Because I fell in love.' `Because you fell in love!' growled Scrooge, as if that were the only one thing in the world more ridiculous than a merry Christmas. `Good afternoon!' `Nay, uncle, but you never came to see me before that happened. Why give it as a reason for not coming now?' `Good afternoon,' said Scrooge. `I want nothing from you; I ask nothing of you; why cannot we be friends?' `Good afternoon,' said Scrooge. Text-Ngrams-2.005/t/03.out0000644000175000017500000000052610350307337014107 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 1-GRAMS (total count: 8) FIRST N-GRAM: The LAST N-GRAM: fox ------------------------ The 1 brown 3 fox 3 quick 1 2-GRAMS (total count: 7) FIRST N-GRAM: The brown LAST N-GRAM: brown fox ------------------------ The brown 1 brown fox 2 brown quick 1 fox brown 2 quick fox 1 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/08.in0000644000175000017500000000012007770361403013707 0ustar vladovladoquick brown fox, brown fox, brown fox, brown fox, and some more text, more text Text-Ngrams-2.005/t/13.t0000644000175000017500000000046112110172616013536 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 1; require 't/auxfunctions.pl'; my $com = "perl -Mblib ./ngrams.pl --n=10 --orderby=frequency --type=character". " --normalize --onlyfirst=100 --spartan t/12.in"; my $out = scalar(`$com`); #putfile('t/13.out-new', $out); &isn('t/13.out', $out); Text-Ngrams-2.005/t/01.t0000644000175000017500000000123510350307337013537 0ustar vladovlado#!/usr/bin/perl -w use Test::More tests => 5; use_ok("Text::Ngrams"); require 't/auxfunctions.pl'; my $ng3 = Text::Ngrams->new; $ng3->feed_tokens('a'); $ng3->feed_tokens('b'); $ng3->feed_tokens('c'); $ng3->feed_tokens('d'); $ng3->feed_tokens('e'); $ng3->feed_tokens('f'); $ng3->feed_tokens('g'); $ng3->feed_tokens('h'); #putfile('t/01.out', $ng3->to_string( 'orderby' => 'ngram' )); isn('t/01.out', $ng3->to_string( 'orderby' => 'ngram' )); is($ng3->{'total_distinct_count'}, 21); #print $ng3->{'total_distinct_count'}."\n"; is(Text::Ngrams::encode_S("abc\n\t\xF6lado"), 'abc\\n\\t^vlado'); is(Text::Ngrams::decode_S('abc\\n\\t^vlado'), "abc\n\t\xF6lado"); Text-Ngrams-2.005/t/04.t0000644000175000017500000000053410350307337013543 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 2; use_ok("Text::Ngrams"); require 't/auxfunctions.pl'; my $ng = Text::Ngrams->new(type=>'byte'); $ng->process_text( ' The brown quick fox, brown fox, brown fox ... '); #putfile('t/04.out', normalize($ng->to_string('orderby' => 'ngram' ))); isn('t/04.out', $ng->to_string('orderby' => 'ngram' )); Text-Ngrams-2.005/t/11.in0000644000175000017500000002335507770361403013720 0ustar vladovladoA CHRISTMAS CAROL I have endeavoured in this Ghostly little book, to raise the Ghost of an Idea, which shall not put my readers out of humour with themselves, with each other, with the season, or with me. May it haunt their houses pleasantly, and no one wish to lay it. Their faithful Friend and Servant, C. D. December, 1843. Stave 1: Marley's Ghost Marley was dead: to begin with. There is no doubt whatever about that. The register of his burial was signed by the clergyman, the clerk, the undertaker, and the chief mourner. Scrooge signed it. And Scrooge's name was good upon `Change, for anything he chose to put his hand to. Old Marley was as dead as a door-nail. Mind! I don't mean to say that I know, of my own knowledge, what there is particularly dead about a door-nail. I might have been inclined, myself, to regard a coffin-nail as the deadest piece of ironmongery in the trade. But the wisdom of our ancestors is in the simile; and my unhallowed hands shall not disturb it, or the Country's done for. You will therefore permit me to repeat, emphatically, that Marley was as dead as a door-nail. Scrooge knew he was dead? Of course he did. How could it be otherwise? Scrooge and he were partners for I don't know how many years. Scrooge was his sole executor, his sole administrator, his sole assign, his sole residuary legatee, his sole friend, and sole mourner. And even Scrooge was not so dreadfully cut up by the sad event, but that he was an excellent man of business on the very day of the funeral, and solemnised it with an undoubted bargain The mention of Marley's funeral brings me back to the point I started from. There is no doubt that Marley was dead. This must be distinctly understood, or nothing wonderful can come of the story I am going to relate. If we were not perfectly convinced that Hamlet's Father died before the play began, there would be nothing more remarkable in his taking a stroll at night, in an easterly wind, upon his own ramparts, than there would be in any other middle-aged gentleman rashly turning out after dark in a breezy spot -- say Saint Paul's Churchyard for instance -- literally to astonish his son's weak mind. Scrooge never painted out Old Marley's name. There it stood, years afterwards, above the warehouse door: Scrooge and Marley. The firm was known as Scrooge and Marley. Sometimes people new to the business called Scrooge Scrooge, and sometimes Marley, but he answered to both names. It was all the same to him. Oh! But he was a tight-fisted hand at the grind-stone, Scrooge! a squeezing, wrenching, grasping, scraping, clutching, covetous, old sinner! Hard and sharp as flint, from which no steel had ever struck out generous fire; secret, and self-contained, and solitary as an oyster. The cold within him froze his old features, nipped his pointed nose, shrivelled his cheek, stiffened his gait; made his eyes red, his thin lips blue and spoke out shrewdly in his grating voice. A frosty rime was on his head, and on his eyebrows, and his wiry chin. He carried his own low temperature always about with him; he iced his office in the dogdays; and didn't thaw it one degree at Christmas. External heat and cold had little influence on Scrooge. No warmth could warm, no wintry weather chill him. No wind that blew was bitterer than he, no falling snow was more intent upon its purpose, no pelting rain less open to entreaty. Foul weather didn't know where to have him. The heaviest rain, and snow, and hail, and sleet, could boast of the advantage over him in only one respect. They often `came down' handsomely, and Scrooge never did. Nobody ever stopped him in the street to say, with gladsome looks, `My dear Scrooge, how are you? When will you come to see me?' No beggars implored him to bestow a trifle, no children asked him what it was o'clock, no man or woman ever once in all his life inquired the way to such and such a place, of Scrooge. Even the blind men's dogs appeared to know him; and when they saw him coming on, would tug their owners into doorways and up courts; and then would wag their tails as though they said, `No eye at all is better than an evil eye, dark master!' But what did Scrooge care! It was the very thing he liked. To edge his way along the crowded paths of life, warning all human sympathy to keep its distance, was what the knowing ones call `nuts' to Scrooge. Once upon a time -- of all the good days in the year, on Christmas Eve -- old Scrooge sat busy in his counting-house. It was cold, bleak, biting weather: foggy withal: and he could hear the people in the court outside, go wheezing up and down, beating their hands upon their breasts, and stamping their feet upon the pavement stones to warm them. The city clocks had only just gone three, but it was quite dark already -- it had not been light all day -- and candles were flaring in the windows of the neighbouring offices, like ruddy smears upon the palpable brown air. The fog came pouring in at every chink and keyhole, and was so dense without, that although the court was of the narrowest, the houses opposite were mere phantoms. To see the dingy cloud come drooping down, obscuring everything, one might have thought that Nature lived hard by, and was brewing on a large scale. The door of Scrooge's counting-house was open that he might keep his eye upon his clerk, who in a dismal little cell beyond, a sort of tank, was copying letters. Scrooge had a very small fire, but the clerk's fire was so very much smaller that it looked like one coal. But he couldn't replenish it, for Scrooge kept the coal-box in his own room; and so surely as the clerk came in with the shovel, the master predicted that it would be necessary for them to part. Wherefore the clerk put on his white comforter, and tried to warm himself at the candle; in which effort, not being a man of a strong imagination, he failed. `A merry Christmas, uncle! God save you!' cried a cheerful voice. It was the voice of Scrooge's nephew, who came upon him so quickly that this was the first intimation he had of his approach. `Bah!' said Scrooge, `Humbug!' He had so heated himself with rapid walking in the fog and frost, this nephew of Scrooge's, that he was all in a glow; his face was ruddy and handsome; his eyes sparkled, and his breath smoked again. `Christmas a humbug, uncle!' said Scrooge's nephew. `You don't mean that, I am sure?' `I do,' said Scrooge. `Merry Christmas! What right have you to be merry? What reason have you to be merry? You're poor enough.' `Come, then,' returned the nephew gaily. `What right have you to be dismal? What reason have you to be morose? You're rich enough.' Scrooge having no better answer ready on the spur of the moment, said `Bah!' again; and followed it up with `Humbug.' `Don't be cross, uncle!' said the nephew. `What else can I be,' returned the uncle, `when I live in such a world of fools as this? Merry Christmas! Out upon merry Christmas! What's Christmas time to you but a time for paying bills without money; a time for finding yourself a year older, but not an hour richer; a time for balancing your books and having every item in `em through a round dozen of months presented dead against you? If I could work my will,' said Scrooge indignantly, `every idiot who goes about with "Merry Christmas" on his lips, should be boiled with his own pudding, and buried with a stake of holly through his heart. He should!' `Uncle!' pleaded the nephew. `Nephew!' returned the uncle sternly, `keep Christmas in your own way, and let me keep it in mine.' `Keep it!' repeated Scrooge's nephew. `But you don't keep it.' `Let me leave it alone, then,' said Scrooge. `Much good may it do you! Much good it has ever done you!' `There are many things from which I might have derived good, by which I have not profited, I dare say,' returned the nephew. `Christmas among the rest. But I am sure I have always thought of Christmas time, when it has come round -- apart from the veneration due to its sacred name and origin, if anything belonging to it can be apart from that -- as a good time; a kind, forgiving, charitable, pleasant time: the only time I know of, in the long calendar of the year, when men and women seem by one consent to open their shut-up hearts freely, and to think of people below them as if they really were fellow-passengers to the grave, and not another race of creatures bound on other journeys. And therefore, uncle, though it has never put a scrap of gold or silver in my pocket, I believe that it has done me good, and will do me good; and I say, God bless it!' The clerk in the Tank involuntarily applauded. Becoming immediately sensible of the impropriety, he poked the fire, and extinguished the last frail spark for ever. `Let me hear another sound from you,' said Scrooge, `and you'll keep your Christmas by losing your situation! You're quite a powerful speaker, sir,' he added, turning to his nephew. `I wonder you don't go into Parliament.' `Don't be angry, uncle. Come! Dine with us tomorrow.' Scrooge said that he would see him -- yes, indeed he did. He went the whole length of the expression, and said that he would see him in that extremity first. `But why?' cried Scrooge's nephew. `Why?' `Why did you get married?' said Scrooge. `Because I fell in love.' `Because you fell in love!' growled Scrooge, as if that were the only one thing in the world more ridiculous than a merry Christmas. `Good afternoon!' `Nay, uncle, but you never came to see me before that happened. Why give it as a reason for not coming now?' `Good afternoon,' said Scrooge. `I want nothing from you; I ask nothing of you; why cannot we be friends?' `Good afternoon,' said Scrooge. Text-Ngrams-2.005/t/03.t0000644000175000017500000000053610350307337013544 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 2; use_ok("Text::Ngrams"); require 't/auxfunctions.pl'; my $ng = Text::Ngrams->new(windowsize=>2, type=>'word'); $ng->process_text('The brown quick fox, brown fox, brown fox ...'); #putfile('t/03.out', normalize($ng->to_string( 'orderby'=>'ngram' ))); isn('t/03.out', $ng->to_string( 'orderby'=>'ngram' )); Text-Ngrams-2.005/t/14.t0000644000175000017500000000040612110173247013537 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 1; require 't/auxfunctions.pl'; my $com = "perl -Mblib ./ngrams.pl --n=3 --orderby=frequency --type=byte". " t/14.in"; my $out = scalar(`$com`); # putfile('t/14.out-new', $out); &isn('t/14.out', $out); Text-Ngrams-2.005/t/05.in0000644000175000017500000000007107770361403013711 0ustar vladovlado The brown quick fox, brown fox, brown fox ... Text-Ngrams-2.005/t/07.out0000644000175000017500000000052612110017531014101 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 1-GRAMS (total count: 8) FIRST N-GRAM: The LAST N-GRAM: fox ------------------------ brown 3 fox 3 The 1 quick 1 2-GRAMS (total count: 7) FIRST N-GRAM: The brown LAST N-GRAM: brown fox ------------------------ brown fox 2 fox brown 2 The brown 1 brown quick 1 quick fox 1 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/15.out0000644000175000017500000000000012110174243014067 0ustar vladovladoText-Ngrams-2.005/t/08.out0000644000175000017500000000045210350307337014112 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 1.8 1-GRAMS (total count: 8) FIRST N-GRAM: quick LAST N-GRAM: text ------------------------ brown 4 fox 4 2-GRAMS (total count: 7) FIRST N-GRAM: quick brown LAST N-GRAM: more text ------------------------ brown fox 4 fox brown 3 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/02.t0000644000175000017500000000066610350307337013547 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 4; BEGIN { use_ok("Text::Ngrams", qw(encode_S decode_S)) } require 't/auxfunctions.pl'; my $ng3 = Text::Ngrams->new; $ng3->process_text('abcdefg1235678hijklmnop'); #putfile('t/02.out', $ng3->to_string( 'orderby' => 'ngram' )); isn('t/02.out', $ng3->to_string( 'orderby'=>'ngram' )); is(encode_S("abc\n\t\xF6lado"), 'abc\\n\\t^vlado'); is(decode_S('abc\\n\\t^vlado'), "abc\n\t\xF6lado"); Text-Ngrams-2.005/t/05.t0000644000175000017500000000154712110016770013543 0ustar vladovlado#!/usr/bin/perl # $Id: 05.t,v 1.3 2006/12/18 22:07:01 vlado Exp $ use Test::More tests => 5; use_ok("Text::Ngrams"); require 't/auxfunctions.pl'; my $ng = Text::Ngrams->new(windowsize=>2, type=>'word'); $ng->process_files('t/05.in'); isn('t/03.out', $ng->to_string('orderby'=>'ngram')); $ng = Text::Ngrams->new(); $ng->process_files('t/05-0.in'); my $o = $ng->to_string( 'orderby' => 'ngram' ); #putfile('t/05-0.out', $o); isn('t/05-0.out', $o); $ng = Text::Ngrams->new(type=>'byte'); $ng->process_files('t/05-1.in'); $o = $ng->to_string( 'orderby' => 'ngram'); #putfile('t/05-1.out', $o); isn('t/05-1.out', $o); $ng = Text::Ngrams->new(type=>'byte'); $ng->process_files('t/05-2.in'); my @a = $ng->get_ngrams( 'orderby' => 'frequency'); $o = ''; while (@a) { my $n=shift @a; my $f = shift @a; $o.= "$n $f\n"; } #putfile('t/05-2.out', $o); isn('t/05-2.out', $o); Text-Ngrams-2.005/t/10.t0000644000175000017500000000044707770361403013551 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 1; require 't/auxfunctions.pl'; my $com = "perl -Mblib ./ngrams.pl --n=2 --orderby=ngram --type=word". " --normalize t/05.in"; my $out = normalize(`$com`); #putfile('t/10.out', $out); is(normalize(scalar(getfile('t/10.out'))), $out); Text-Ngrams-2.005/t/13.out0000644000175000017500000000776412110172611014112 0ustar vladovladoD _ S C R O O G E _ 0.0025792188651437 _ C H R I S T M A S 0.00239498894620486 C H R I S T M A S _ 0.00239498894620486 I D _ S C R O O G E 0.00184229918938836 _ S A I D _ S C R O 0.00165806927044952 A I D _ S C R O O G 0.00165806927044952 S A I D _ S C R O O 0.00165806927044952 _ S C R O O G E _ S 0.00147383935151069 _ M E R R Y _ C H R 0.00110537951363301 E R R Y _ C H R I S 0.00110537951363301 M E R R Y _ C H R I 0.00110537951363301 R R Y _ C H R I S T 0.00110537951363301 R Y _ C H R I S T M 0.00110537951363301 S C R O O G E _ S _ 0.00110537951363301 Y _ C H R I S T M A 0.00110537951363301 _ H A V E _ Y O U _ 0.000736919675755343 _ R E T U R N E D _ 0.000736919675755343 _ S _ N E P H E W _ 0.000736919675755343 _ T H E _ C L E R K 0.000736919675755343 _ T H E _ N E P H E 0.000736919675755343 _ Y O U _ T O _ B E 0.000736919675755343 A V E _ Y O U _ T O 0.000736919675755343 C R O O G E _ S _ N 0.000736919675755343 D _ T H E _ N E P H 0.000736919675755343 E _ S _ N E P H E W 0.000736919675755343 E _ Y O U _ T O _ B 0.000736919675755343 E T U R N E D _ T H 0.000736919675755343 G E _ S _ N E P H E 0.000736919675755343 H A V E _ Y O U _ T 0.000736919675755343 H E _ N E P H E W _ 0.000736919675755343 I G H T _ H A V E _ 0.000736919675755343 O G E _ S _ N E P H 0.000736919675755343 O O G E _ S _ N E P 0.000736919675755343 R E T U R N E D _ T 0.000736919675755343 R O O G E _ S _ N E 0.000736919675755343 T H E _ C L E R K _ 0.000736919675755343 T H E _ N E P H E W 0.000736919675755343 T U R N E D _ T H E 0.000736919675755343 U R N E D _ T H E _ 0.000736919675755343 V E _ Y O U _ T O _ 0.000736919675755343 Y O U _ T O _ B E _ 0.000736919675755343 _ A _ T I M E _ F O 0.000552689756816507 _ A F T E R N O O N 0.000552689756816507 _ G O O D _ A F T E 0.000552689756816507 _ N E P H E W _ W H 0.000552689756816507 _ O F _ S C R O O G 0.000552689756816507 _ S C R O O G E _ H 0.000552689756816507 _ T H A T _ H E _ W 0.000552689756816507 _ T I M E _ F O R _ 0.000552689756816507 _ Y O U _ D O N _ T 0.000552689756816507 A _ T I M E _ F O R 0.000552689756816507 A F T E R N O O N _ 0.000552689756816507 D _ A F T E R N O O 0.000552689756816507 E D _ S C R O O G E 0.000552689756816507 E D _ T H E _ N E P 0.000552689756816507 F _ S C R O O G E _ 0.000552689756816507 G O O D _ A F T E R 0.000552689756816507 I N G _ I N _ T H E 0.000552689756816507 N _ S A I D _ S C R 0.000552689756816507 N G _ I N _ T H E _ 0.000552689756816507 O D _ A F T E R N O 0.000552689756816507 O F _ S C R O O G E 0.000552689756816507 O O D _ A F T E R N 0.000552689756816507 O U _ T O _ B E _ M 0.000552689756816507 Y O U _ D O N _ T _ 0.000552689756816507 _ A _ M E R R Y _ C 0.000368459837877671 _ A M _ S U R E _ I 0.000368459837877671 _ A P A R T _ F R O 0.000368459837877671 _ B E _ M E R R Y _ 0.000368459837877671 _ C O U N T I N G _ 0.000368459837877671 _ D O N _ T _ B E _ 0.000368459837877671 _ F E L L _ I N _ L 0.000368459837877671 _ F R O M _ Y O U _ 0.000368459837877671 _ G O O D _ A N D _ 0.000368459837877671 _ H E _ W O U L D _ 0.000368459837877671 _ I _ A M _ S U R E 0.000368459837877671 _ I T _ W A S _ T H 0.000368459837877671 _ M E _ G O O D _ A 0.000368459837877671 _ M I G H T _ H A V 0.000368459837877671 _ M U C H _ G O O D 0.000368459837877671 _ R E A S O N _ H A 0.000368459837877671 _ R I G H T _ H A V 0.000368459837877671 _ S A I D _ T H A T 0.000368459837877671 _ S C R O O G E _ A 0.000368459837877671 _ S C R O O G E _ I 0.000368459837877671 _ S C R O O G E _ M 0.000368459837877671 _ T H E _ C O U R T 0.000368459837877671 _ T H E _ O N L Y _ 0.000368459837877671 _ T H E _ U N C L E 0.000368459837877671 _ T H E _ Y E A R _ 0.000368459837877671 _ T O _ B E _ M E R 0.000368459837877671 _ U N C L E _ S A I 0.000368459837877671 _ U P O N _ T H E _ 0.000368459837877671 _ W A S _ T H E _ V 0.000368459837877671 _ W H A T _ R E A S 0.000368459837877671 _ W H A T _ R I G H 0.000368459837877671 _ W O U L D _ S E E 0.000368459837877671 A _ M E R R Y _ C H 0.000368459837877671 A I D _ T H A T _ H 0.000368459837877671 A M _ S U R E _ I _ 0.000368459837877671 Text-Ngrams-2.005/t/14.out0000644000175000017500000000176312110173242014105 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 2.004 1-GRAMS (total count: 58) FIRST N-GRAM: f LAST N-GRAM: \n ------------------------- _ 7 i 7 n 6 e 5 l 5 s 4 \n 3 d 3 t 3 \r 2 a 2 c 2 o 2 r 2 f 1 h 1 m 1 u 1 x 1 2-GRAMS (total count: 57) FIRST N-GRAM: f i LAST N-GRAM: e \n ------------------------- _ l 4 i n 4 l i 4 n e 4 e _ 3 d _ 2 i r 2 s t 2 t _ 2 \n s 1 \n t 1 \r \n 1 \r l 1 _ d 1 _ m 1 _ u 1 a c 1 a s 1 c \r 1 c o 1 d o 1 e \n 1 e c 1 f i 1 h i 1 i x 1 l a 1 m a 1 n d 1 n i 1 o n 1 o s 1 r d 1 r s 1 s \r 1 s e 1 t h 1 u n 1 x \n 1 3-GRAMS (total count: 56) FIRST N-GRAM: f i r LAST N-GRAM: n e \n ------------------------- _ l i 4 i n e 4 l i n 4 n e _ 3 d _ l 2 s t _ 2 t _ l 2 \n s e 1 \n t h 1 \r \n t 1 \r l a 1 _ d o 1 _ m a 1 _ u n 1 a c \r 1 a s t 1 c \r l 1 c o n 1 d o s 1 e _ d 1 e _ m 1 e _ u 1 e c o 1 f i r 1 h i r 1 i r d 1 i r s 1 i x \n 1 l a s 1 m a c 1 n d _ 1 n e \n 1 n i x 1 o n d 1 o s \r 1 r d _ 1 r s t 1 s \r \n 1 s e c 1 t h i 1 u n i 1 x \n s 1 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/05-0.out0000644000175000017500000000072410350310204014231 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 1.8 1-GRAMS (total count: 12) FIRST N-GRAM: M LAST N-GRAM: _ ------------------------- _ 1 I 4 M 1 P 2 S 4 2-GRAMS (total count: 11) FIRST N-GRAM: M I LAST N-GRAM: I _ ------------------------- I _ 1 I P 1 I S 2 M I 1 P I 1 P P 1 S I 2 S S 2 3-GRAMS (total count: 10) FIRST N-GRAM: M I S LAST N-GRAM: P I _ ------------------------- I P P 1 I S S 2 M I S 1 P I _ 1 P P I 1 S I P 1 S I S 1 S S I 2 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/07.t0000644000175000017500000000152312110171406013535 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 2; use_ok("Text::Ngrams"); require 't/auxfunctions.pl'; my $ng = Text::Ngrams->new(windowsize=>2, type=>'word'); $ng->process_files('t/05.in'); my $producedout = normalize($ng->to_string( orderby=>'frequency' )); # putfile('t/07.out-new', $producedout); my $oldout = normalize(scalar(getfile('t/07.out'))); # ordering may vary, so let us normalize it further $producedout = &normalize1( $producedout ); $oldout = &normalize1( $oldout ); is($producedout, $oldout); sub normalize1 { my $s = shift; my $r; while ($s) { if ($s =~ /\n---+\n/) { $r .= "$`$&"; $s = $'; } else { return "$r$s" } while ($s =~ /^\S+\t(\d+)\n/) { my $n = $1; my @a; while ($s =~ /^\S+\t$n\n/) { push @a, $&; $s = $'; } $r .= join('',sort(@a)); } } return $r; } Text-Ngrams-2.005/t/05-2.out0000644000175000017500000000041412110016763014240 0ustar vladovlado0 1 2 2 1 2 3 2 i s _ 2 i s s 2 s s i 2 \n M i 1 _ 0 0 1 _ 0 1 1 _ a _ 1 _ i s 1 _ t e 1 0 0 1 1 2 3 \n 1 2 3 4 1 3 \n M 1 3 4 _ 1 4 _ 0 1 M i s 1 T h i 1 a _ t 1 e s t 1 h i s 1 i . \n 1 i p p 1 p i . 1 p p i 1 s _ a 1 s _ i 1 s i p 1 s i s 1 s t _ 1 t _ 0 1 t e s 1 Text-Ngrams-2.005/t/05-2.in0000644000175000017500000000005012110013356014026 0ustar vladovladoThis is a test 01234 00123 Mississippi. Text-Ngrams-2.005/t/auxfunctions.pl0000644000175000017500000000140510350307337016214 0ustar vladovlado#!/usr/bin/perl sub isn { my $get = normalize(scalar(getfile($_[0]))); my $new = normalize($_[1]); return is($get, $new); } sub getfile($) { my $f = shift; local *F; open(F, "<$f") or die "getfile:cannot open $f:$!"; my @r = ; close(F); return wantarray ? @r : join ('', @r); } sub putfile($@) { my $f = shift; local *F; open(F, ">$f") or die "putfile:cannot open $f:$!"; print F '' unless @_; while (@_) { print F shift(@_) } close(F); } sub normalize { my $r = shift; $r =~ s/(BEGIN OUTPUT BY Text::Ngrams version )[\d.]+/$1/; $r =~ s/(\s\d\.\d\d\d\d\d\d\d\d\d\d\d\d\d\d)\d*/$1/g; # used sometimes #$r =~ s/[ _]//g; #$r = join("\n", sort(split(/\n/, $r))); return $r; } 1; Text-Ngrams-2.005/t/11.out0000644000175000017500000007610012110171704014100 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 2.004 1-GRAMS (total count: 9965) FIRST N-GRAM: A LAST N-GRAM: \n --------------------------- _ 0.179729051680883 e 0.0903161063723031 o 0.0635223281485198 t 0.0586051179126944 a 0.0572002007024586 n 0.0509784244856999 i 0.0466633216256899 h 0.045960863020572 r 0.0457601605619669 s 0.0421475163070748 d 0.0334169593577521 l 0.0286001003512293 \n 0.0250878073256397 u 0.0221776216758655 w 0.0188660311088811 m 0.0187656798795785 c 0.0184646261916708 g 0.0184646261916708 y 0.017059708981435 , 0.0149523331660813 f 0.0133467134972403 p 0.0124435524335173 b 0.00943301555444054 . 0.00832915203211239 ' 0.00742599096838936 v 0.00662318113396889 k 0.0061214249874561 S 0.00441545408931259 \` 0.00441545408931259 I 0.00321123933768189 - 0.00311088810837933 ! 0.00270948319116909 C 0.0022077270446563 T 0.00200702458605118 M 0.00190667335674862 ; 0.00180632212744606 ? 0.00160561966884094 B 0.00120421475163071 W 0.00110386352232815 H 0.00100351229302559 A 0.000802809834420472 G 0.000802809834420472 N 0.000802809834420472 O 0.000702458605117913 x 0.000702458605117913 : 0.000602107375815354 D 0.000501756146512795 Y 0.000501756146512795 q 0.000501756146512795 z 0.000501756146512795 E 0.000301053687907677 F 0.000301053687907677 L 0.000301053687907677 " 0.000200702458605118 1 0.000200702458605118 P 0.000200702458605118 R 0.000200702458605118 j 0.000200702458605118 3 0.000100351229302559 4 0.000100351229302559 8 0.000100351229302559 K 0.000100351229302559 U 0.000100351229302559 2-GRAMS (total count: 9964) FIRST N-GRAM: A _ LAST N-GRAM: . \n --------------------------- e _ 0.0261942994781212 _ _ 0.0201726214371738 t h 0.0183661180248896 _ t 0.0177639502207949 h e 0.0172621437173826 d _ 0.0164592533119229 _ a 0.0153552790044159 s _ 0.0145523885989562 t _ 0.0141509433962264 i n 0.0139502207948615 , _ 0.0121437173825773 _ h 0.0120433560818948 e r 0.0113408269771176 n _ 0.0109393817743878 a n 0.0108390204737053 _ w 0.0107386591730229 o u 0.00993576876756323 r e 0.00933360096346849 _ s 0.00913287836210357 _ o 0.0088317944600562 n d 0.0088317944600562 _ i 0.00863107185869129 h i 0.00843034925732638 a s 0.00832998795664392 h a 0.008129265355279 o n 0.00782818145323163 y _ 0.00782818145323163 i s 0.00772782015254918 l e 0.00772782015254918 _ b 0.00692492974708952 i t 0.00692492974708952 r o 0.00692492974708952 o o 0.00682456844640707 a t 0.00672420714572461 a r 0.00662384584504215 e d 0.00652348454435969 n g 0.00642312324367724 _ c 0.00632276194299478 o _ 0.00622240064231232 s t 0.00612203934162987 m e 0.00602167804094741 r _ 0.00602167804094741 n e 0.0058209554395825 e a 0.00572059413890004 _ d 0.00541951023685267 t o 0.00541951023685267 _ m 0.00531914893617021 v e 0.00531914893617021 e n 0.0051184263348053 . _ 0.00491770373344039 g e 0.00491770373344039 _ f 0.00481734243275793 c r 0.00471698113207547 w a 0.00471698113207547 \n \n 0.00461661983139301 o r 0.00461661983139301 _ \` 0.0044158972300281 n o 0.0044158972300281 _ n 0.00431553592934564 o g 0.00431553592934564 r i 0.00431553592934564 f _ 0.00421517462866319 \n _ 0.00411481332798073 t e 0.00411481332798073 o f 0.00401445202729827 g _ 0.00391409072661582 o w 0.00391409072661582 S c 0.00381372942593336 a l 0.00381372942593336 u r 0.00381372942593336 h _ 0.0037133681252509 _ S 0.00361300682456845 a i 0.00361300682456845 n t 0.00361300682456845 u t 0.00361300682456845 l l 0.00351264552388599 m a 0.00351264552388599 s e 0.00351264552388599 i m 0.00341228422320353 _ p 0.00331192292252108 w i 0.00331192292252108 _ r 0.00321156162183862 b e 0.00321156162183862 e , 0.00321156162183862 l _ 0.00321156162183862 c o 0.00311120032115616 e s 0.00311120032115616 _ e 0.00291047771979125 _ y 0.00291047771979125 a _ 0.00291047771979125 d e 0.00291047771979125 e e 0.00291047771979125 o m 0.00291047771979125 t i 0.00291047771979125 _ I 0.00281011641910879 a d 0.00281011641910879 d o 0.00281011641910879 e \n 0.00281011641910879 h o 0.00281011641910879 l d 0.00281011641910879 3-GRAMS (total count: 9963) FIRST N-GRAM: A _ C LAST N-GRAM: e . \n --------------------------- _ _ _ 0.0158586771052896 _ t h 0.0127471645086821 t h e 0.0105389942788317 h e _ 0.00923416641573823 _ a n 0.00612265381913078 a n d 0.00592191107096256 n d _ 0.00572116832279434 i n g 0.00542005420054201 _ h i 0.00511894007828967 a s _ 0.00491819733012145 e d _ 0.004516711833785 i s _ 0.00431596908561678 _ i n 0.00421559771153267 h i s 0.00421559771153267 \n _ _ 0.00411522633744856 c r o 0.00401485496336445 r o o 0.00401485496336445 \n \n _ 0.00391448358928034 _ w a 0.00391448358928034 S c r 0.00381411221519623 i n _ 0.00381411221519623 o g e 0.00381411221519623 o o g 0.00381411221519623 n g _ 0.003613369467028 t o _ 0.00351299809294389 _ t o 0.00341262671885978 h a t 0.00341262671885978 _ S c 0.00331225534477567 _ h a 0.00331225534477567 a t _ 0.00331225534477567 o f _ 0.00331225534477567 _ o f 0.00321188397069156 w a s 0.00311151259660745 _ b e 0.00301114122252334 _ i t 0.00301114122252334 t h a 0.00281039847435511 _ h e 0.002710027100271 e , _ 0.002710027100271 m e _ 0.002710027100271 _ a _ 0.00260965572618689 _ c o 0.00260965572618689 e _ w 0.00260965572618689 e r _ 0.00260965572618689 u t _ 0.00260965572618689 , _ a 0.00250928435210278 n _ t 0.00250928435210278 o n _ 0.00250928435210278 r e _ 0.00250928435210278 y o u 0.00250928435210278 _ w i 0.00240891297801867 _ n o 0.00230854160393456 _ s a 0.00230854160393456 _ y o 0.00230854160393456 _ _ \` 0.00220817022985045 _ d o 0.00220817022985045 _ o n 0.00220817022985045 a n _ 0.00220817022985045 d _ t 0.00220817022985045 e _ c 0.00220817022985045 h e r 0.00220817022985045 _ m e 0.00210779885576634 d _ h 0.00210779885576634 i t _ 0.00210779885576634 l d _ 0.00210779885576634 t _ h 0.00210779885576634 _ I _ 0.00200742748168222 . \n \n 0.00200742748168222 e r e 0.00200742748168222 i s t 0.00200742748168222 i t h 0.00200742748168222 v e r 0.00200742748168222 e _ i 0.00190705610759811 f o r 0.00190705610759811 l l _ 0.00190705610759811 s _ a 0.00190705610759811 w i t 0.00190705610759811 _ r e 0.001806684733514 _ w h 0.001806684733514 ' \n \n 0.001806684733514 a l l 0.001806684733514 e _ t 0.001806684733514 g e _ 0.001806684733514 l y _ 0.001806684733514 o u r 0.001806684733514 r y _ 0.001806684733514 t e r 0.001806684733514 v e _ 0.001806684733514 _ a s 0.00170631335942989 d , _ 0.00170631335942989 e v e 0.00170631335942989 h i n 0.00170631335942989 n _ h 0.00170631335942989 n o t 0.00170631335942989 t h _ 0.00170631335942989 t h i 0.00170631335942989 _ s o 0.00160594198534578 a v e 0.00160594198534578 c l e 0.00160594198534578 d _ S 0.00160594198534578 e _ a 0.00160594198534578 4-GRAMS (total count: 9962) FIRST N-GRAM: A _ C H LAST N-GRAM: g e . \n --------------------------- _ _ _ _ 0.0117446295924513 _ t h e 0.00903433045573178 t h e _ 0.00652479421802851 a n d _ 0.00481830957639028 _ a n d 0.00451716522786589 \n _ _ _ 0.00411563942983337 \n \n _ _ 0.0039148765308171 S c r o 0.00381449508130897 c r o o 0.00381449508130897 h i s _ 0.00381449508130897 o o g e 0.00381449508130897 r o o g 0.00381449508130897 _ h i s 0.00361373218229271 _ S c r 0.00331258783376832 i n g _ 0.00321220638426019 _ i n _ 0.00311182493475206 _ t o _ 0.00281068058622767 _ w a s 0.00281068058622767 _ o f _ 0.00271029913671953 w a s _ 0.00271029913671953 h a t _ 0.00250953623770327 _ y o u 0.00230877333868701 n _ t h 0.00230877333868701 _ _ _ \` 0.00220839188917888 , _ a n 0.00220839188917888 t h a t 0.00220839188917888 _ t h a 0.00210801043967075 _ i t _ 0.00190724754065449 . \n \n _ 0.00190724754065449 w i t h 0.00190724754065449 ' \n \n _ 0.00180686609114636 _ h e _ 0.00160610319213009 o g e _ 0.00160610319213009 _ h i m 0.00150572174262196 _ w i t 0.00150572174262196 d _ S c 0.00150572174262196 d _ t h 0.00150572174262196 h e _ c 0.00150572174262196 o u l d 0.00150572174262196 t h e r 0.00150572174262196 _ b e _ 0.00140534029311383 _ s a i 0.00140534029311383 C h r i 0.00140534029311383 e v e r 0.00140534029311383 h r i s 0.00140534029311383 i s t m 0.00140534029311383 r i s t 0.00140534029311383 s a i d 0.00140534029311383 s t m a 0.00140534029311383 t _ t h 0.00140534029311383 t m a s 0.00140534029311383 _ T h e 0.0013049588436057 _ a s _ 0.0013049588436057 _ h a v 0.0013049588436057 a v e _ 0.0013049588436057 h e _ w 0.0013049588436057 i n _ t 0.0013049588436057 i t h _ 0.0013049588436057 t h i n 0.0013049588436057 _ C h r 0.00120457739409757 _ n o t 0.00120457739409757 , _ t h 0.00120457739409757 a i d _ 0.00120457739409757 a s _ a 0.00120457739409757 e _ w a 0.00120457739409757 e d _ h 0.00120457739409757 e d _ t 0.00120457739409757 e r e _ 0.00120457739409757 h i n g 0.00120457739409757 n _ h i 0.00120457739409757 t _ h a 0.00120457739409757 _ c o u 0.00110419594458944 _ f o r 0.00110419594458944 e _ i n 0.00110419594458944 e p h e 0.00110419594458944 h a v e 0.00110419594458944 h e r e 0.00110419594458944 p h e w 0.00110419594458944 t i m e 0.00110419594458944 u l d _ 0.00110419594458944 v e r _ 0.00110419594458944 _ f r o 0.00100381449508131 ' _ s a 0.00100381449508131 . _ T h 0.00100381449508131 M a r l 0.00100381449508131 a l l _ 0.00100381449508131 a r l e 0.00100381449508131 d _ h i 0.00100381449508131 f _ t h 0.00100381449508131 f o r _ 0.00100381449508131 n d _ s 0.00100381449508131 n e p h 0.00100381449508131 n o t _ 0.00100381449508131 o f _ t 0.00100381449508131 o n _ h 0.00100381449508131 p o n _ 0.00100381449508131 r l e y 0.00100381449508131 u p o n 0.00100381449508131 _ M a r 0.000903433045573178 _ d e a 0.000903433045573178 5-GRAMS (total count: 9961) FIRST N-GRAM: A _ C H R LAST N-GRAM: o g e . \n --------------------------- _ _ _ _ _ 0.0076297560485895 _ t h e _ 0.00612388314426262 \n _ _ _ _ 0.00411605260516012 _ a n d _ 0.004015661078205 \n \n _ _ _ 0.00391526955124987 S c r o o 0.00381487802429475 c r o o g 0.00381487802429475 r o o g e 0.00381487802429475 _ S c r o 0.00331292038951912 _ h i s _ 0.00331292038951912 _ w a s _ 0.002409396646923 _ _ _ _ \` 0.00220861359301275 , _ a n d 0.00220861359301275 n _ t h e 0.0020078305391025 . \n \n _ _ 0.00190743901214737 _ t h a t 0.00180704748519225 ' \n \n _ _ 0.00180704748519225 t h a t _ 0.00180704748519225 o o g e _ 0.001606264431282 _ w i t h 0.00150587290432687 d _ S c r 0.00150587290432687 _ s a i d 0.00140548137737175 C h r i s 0.00140548137737175 h r i s t 0.00140548137737175 i s t m a 0.00140548137737175 r i s t m 0.00140548137737175 s t m a s 0.00140548137737175 i n _ t h 0.00130508985041662 w i t h _ 0.00130508985041662 _ C h r i 0.0012046983234615 s a i d _ 0.0012046983234615 _ h a v e 0.00110430679650637 _ i n _ t 0.00110430679650637 d _ t h e 0.00110430679650637 e p h e w 0.00110430679650637 o u l d _ 0.00110430679650637 ' _ s a i 0.00100391526955125 M a r l e 0.00100391526955125 a n d _ s 0.00100391526955125 a r l e y 0.00100391526955125 e _ w a s 0.00100391526955125 f _ t h e 0.00100391526955125 h a v e _ 0.00100391526955125 n _ h i s 0.00100391526955125 n e p h e 0.00100391526955125 t h e _ c 0.00100391526955125 t h i n g 0.00100391526955125 u p o n _ 0.00100391526955125 _ M a r l 0.000903523742596125 _ T h e _ 0.000903523742596125 _ f o r _ 0.000903523742596125 _ u p o n 0.000903523742596125 . _ T h e 0.000903523742596125 e d _ t h 0.000903523742596125 e v e r _ 0.000903523742596125 i d _ S c 0.000903523742596125 o f _ t h 0.000903523742596125 _ d e a d 0.000803132215641 _ n e p h 0.000803132215641 _ n o t _ 0.000803132215641 _ o f _ t 0.000803132215641 _ t h e \n 0.000803132215641 . ' \n \n _ 0.000803132215641 a i d _ S 0.000803132215641 e _ t o _ 0.000803132215641 e d _ h i 0.000803132215641 h e i r _ 0.000803132215641 i n g , _ 0.000803132215641 o o g e . 0.000803132215641 t _ h e _ 0.000803132215641 t _ t h e 0.000803132215641 u n c l e 0.000803132215641 _ e v e r 0.000702740688685875 _ f r o m 0.000702740688685875 _ h e _ w 0.000702740688685875 _ h i m _ 0.000702740688685875 _ i n _ a 0.000702740688685875 _ t h e i 0.000702740688685875 _ u n c l 0.000702740688685875 _ y o u _ 0.000702740688685875 , _ t h e 0.000702740688685875 ; _ a n d 0.000702740688685875 a n d _ h 0.000702740688685875 d _ h i s 0.000702740688685875 e _ i n _ 0.000702740688685875 e _ y o u 0.000702740688685875 h e r e _ 0.000702740688685875 i g h t _ 0.000702740688685875 o g e ' s 0.000702740688685875 o n _ h i 0.000702740688685875 o n ' t _ 0.000702740688685875 o o g e ' 0.000702740688685875 p h e w . 0.000702740688685875 t _ w a s 0.000702740688685875 t h e _ s 0.000702740688685875 t h e i r 0.000702740688685875 v e r y _ 0.000702740688685875 w o u l d 0.000702740688685875 _ c l e r 0.00060234916173075 _ c o u l 0.00060234916173075 6-GRAMS (total count: 9960) FIRST N-GRAM: A _ C H R I LAST N-GRAM: o o g e . \n --------------------------- \n _ _ _ _ _ 0.00411646586345382 \n \n _ _ _ _ 0.00391566265060241 S c r o o g 0.00381526104417671 c r o o g e 0.00381526104417671 _ _ _ _ _ _ 0.0035140562248996 _ S c r o o 0.00331325301204819 _ _ _ _ _ \` 0.00220883534136546 , _ a n d _ 0.00200803212851406 . \n \n _ _ _ 0.00190763052208835 ' \n \n _ _ _ 0.00180722891566265 r o o g e _ 0.00160642570281125 d _ S c r o 0.00150602409638554 _ t h a t _ 0.00140562248995984 C h r i s t 0.00140562248995984 h r i s t m 0.00140562248995984 i s t m a s 0.00140562248995984 n _ t h e _ 0.00140562248995984 r i s t m a 0.00140562248995984 _ C h r i s 0.00120481927710843 _ s a i d _ 0.00120481927710843 _ i n _ t h 0.00110441767068273 i n _ t h e 0.00110441767068273 _ h a v e _ 0.00100401606425703 _ w i t h _ 0.00100401606425703 ' _ s a i d 0.00100401606425703 M a r l e y 0.00100401606425703 d _ t h e _ 0.00100401606425703 n e p h e w 0.00100401606425703 _ M a r l e 0.000903614457831325 _ u p o n _ 0.000903614457831325 e _ w a s _ 0.000903614457831325 i d _ S c r 0.000903614457831325 n _ h i s _ 0.000903614457831325 o f _ t h e 0.000903614457831325 _ n e p h e 0.000803212851405623 _ t h e _ c 0.000803212851405623 . ' \n \n _ _ 0.000803212851405623 a i d _ S c 0.000803212851405623 e d _ t h e 0.000803212851405623 f _ t h e _ 0.000803212851405623 r o o g e . 0.000803212851405623 s a i d _ S 0.000803212851405623 _ a n d _ s 0.00070281124497992 _ o f _ t h 0.00070281124497992 _ t h e _ s 0.00070281124497992 _ t h e i r 0.00070281124497992 _ u n c l e 0.00070281124497992 e p h e w . 0.00070281124497992 o o g e ' s 0.00070281124497992 r o o g e ' 0.00070281124497992 t _ w a s _ 0.00070281124497992 t h e i r _ 0.00070281124497992 _ a n d _ h 0.000602409638554217 _ c o u l d 0.000602409638554217 _ f r o m _ 0.000602409638554217 _ h i s _ o 0.000602409638554217 _ h i s _ s 0.000602409638554217 _ t h e _ w 0.000602409638554217 _ w o u l d 0.000602409638554217 ! ' \n \n _ _ 0.000602409638554217 , ' _ s a i 0.000602409638554217 . _ T h e _ 0.000602409638554217 d _ h i s _ 0.000602409638554217 e r r y _ C 0.000602409638554217 h i s _ s o 0.000602409638554217 o n _ h i s 0.000602409638554217 r r y _ C h 0.000602409638554217 r y _ C h r 0.000602409638554217 t _ t h e _ 0.000602409638554217 t h a t _ h 0.000602409638554217 t h i n g _ 0.000602409638554217 w o u l d _ 0.000602409638554217 y _ C h r i 0.000602409638554217 \n S c r o o 0.000502008032128514 _ a f t e r 0.000502008032128514 _ a n d _ w 0.000502008032128514 _ c l e r k 0.000502008032128514 _ i t _ h a 0.000502008032128514 _ k e e p _ 0.000502008032128514 _ m e r r y 0.000502008032128514 _ t h e _ n 0.000502008032128514 _ t h e r e 0.000502008032128514 _ t i m e _ 0.000502008032128514 _ w a s _ a 0.000502008032128514 _ w h i c h 0.000502008032128514 , _ h i s _ 0.000502008032128514 , _ u n c l 0.000502008032128514 ; _ a n d _ 0.000502008032128514 a s _ d e a 0.000502008032128514 a t _ h e _ 0.000502008032128514 a v e _ y o 0.000502008032128514 d , _ a n d 0.000502008032128514 d o n ' t _ 0.000502008032128514 e _ c l e r 0.000502008032128514 e _ t h e _ 0.000502008032128514 e d _ h i s 0.000502008032128514 g h t _ h a 0.000502008032128514 h a t _ h e 0.000502008032128514 h e _ c l e 0.000502008032128514 h e w . _ \` 0.000502008032128514 7-GRAMS (total count: 9959) FIRST N-GRAM: A _ C H R I S LAST N-GRAM: r o o g e . \n --------------------------- \n \n _ _ _ _ _ 0.00391605582889848 S c r o o g e 0.00381564414097801 _ _ _ _ _ _ _ 0.00341399738929611 _ S c r o o g 0.00331358570137564 \n _ _ _ _ _ \` 0.00220905713425043 . \n \n _ _ _ _ 0.001907822070489 ' \n \n _ _ _ _ 0.00180741038256853 c r o o g e _ 0.00160658700672758 d _ S c r o o 0.00150617531880711 C h r i s t m 0.00140576363088664 h r i s t m a 0.00140576363088664 r i s t m a s 0.00140576363088664 _ C h r i s t 0.00120494025504569 i n _ t h e _ 0.00100411687920474 _ M a r l e y 0.000903705191284266 _ i n _ t h e 0.000903705191284266 ' _ s a i d _ 0.000903705191284266 i d _ S c r o 0.000903705191284266 _ n e p h e w 0.000803293503363792 _ s a i d _ S 0.000803293503363792 . ' \n \n _ _ _ 0.000803293503363792 a i d _ S c r 0.000803293503363792 c r o o g e . 0.000803293503363792 e d _ t h e _ 0.000803293503363792 o f _ t h e _ 0.000803293503363792 s a i d _ S c 0.000803293503363792 _ o f _ t h e 0.000702881815443318 _ t h e i r _ 0.000702881815443318 c r o o g e ' 0.000702881815443318 n e p h e w . 0.000702881815443318 r o o g e ' s 0.000702881815443318 _ h i s _ s o 0.000602470127522844 ! ' \n \n _ _ _ 0.000602470127522844 , ' _ s a i d 0.000602470127522844 e r r y _ C h 0.000602470127522844 o n _ h i s _ 0.000602470127522844 r r y _ C h r 0.000602470127522844 r y _ C h r i 0.000602470127522844 y _ C h r i s 0.000602470127522844 \n S c r o o g 0.00050205843960237 _ t h e _ n e 0.00050205843960237 _ w h i c h _ 0.00050205843960237 _ w o u l d _ 0.00050205843960237 , _ a n d _ s 0.00050205843960237 , _ u n c l e 0.00050205843960237 a s _ d e a d 0.00050205843960237 a v e _ y o u 0.00050205843960237 c r o o g e , 0.00050205843960237 e d _ h i s _ 0.00050205843960237 e p h e w . _ 0.00050205843960237 g h t _ h a v 0.00050205843960237 h a t _ h e _ 0.00050205843960237 h e _ c l e r 0.00050205843960237 h i s _ s o l 0.00050205843960237 h t _ h a v e 0.00050205843960237 i g h t _ h a 0.00050205843960237 i s _ s o l e 0.00050205843960237 p h e w . _ \` 0.00050205843960237 r o o g e , _ 0.00050205843960237 t h a t _ h e 0.00050205843960237 y , _ a n d _ 0.00050205843960237 \n _ _ _ _ _ S 0.000401646751681896 \n _ _ _ _ _ T 0.000401646751681896 _ _ _ _ _ S c 0.000401646751681896 _ _ _ _ _ T h 0.000401646751681896 _ _ _ _ _ \` B 0.000401646751681896 _ _ _ _ S c r 0.000401646751681896 _ _ _ _ T h e 0.000401646751681896 _ _ _ S c r o 0.000401646751681896 _ _ S c r o o 0.000401646751681896 _ I t _ w a s 0.000401646751681896 _ a _ t i m e 0.000401646751681896 _ c o u l d _ 0.000401646751681896 _ d e a d _ a 0.000401646751681896 _ h a v e _ y 0.000401646751681896 _ h e _ w a s 0.000401646751681896 _ h i s _ o w 0.000401646751681896 _ i n _ h i s 0.000401646751681896 _ i t _ h a s 0.000401646751681896 _ m i g h t _ 0.000401646751681896 _ n e v e r _ 0.000401646751681896 _ o n _ h i s 0.000401646751681896 _ r e t u r n 0.000401646751681896 _ t h a t _ h 0.000401646751681896 _ t h o u g h 0.000401646751681896 ' _ r e t u r 0.000401646751681896 , _ h i s _ s 0.000401646751681896 ? ' \n \n _ _ _ 0.000401646751681896 I t _ w a s _ 0.000401646751681896 a _ t i m e _ 0.000401646751681896 a t _ h e _ w 0.000401646751681896 d _ M a r l e 0.000401646751681896 d _ t h e _ n 0.000401646751681896 d , _ a n d _ 0.000401646751681896 e _ c l e r k 0.000401646751681896 e _ n e p h e 0.000401646751681896 e , _ a n d _ 0.000401646751681896 e . \n \n _ _ _ 0.000401646751681896 e d _ S c r o 0.000401646751681896 e t u r n e d 0.000401646751681896 8-GRAMS (total count: 9958) FIRST N-GRAM: A _ C H R I S T LAST N-GRAM: c r o o g e . \n --------------------------- _ _ _ _ _ _ _ _ 0.00331391845752159 _ S c r o o g e 0.00331391845752159 \n \n _ _ _ _ _ \` 0.00220927897168106 . \n \n _ _ _ _ _ 0.00190801365736092 ' \n \n _ _ _ _ _ 0.00180759188592087 S c r o o g e _ 0.00160674834304077 d _ S c r o o g 0.00150632657160072 C h r i s t m a 0.00140590480016067 h r i s t m a s 0.00140590480016067 _ C h r i s t m 0.00120506125728058 i d _ S c r o o 0.000903795942960434 _ i n _ t h e _ 0.000803374171520386 _ s a i d _ S c 0.000803374171520386 ' _ s a i d _ S 0.000803374171520386 . ' \n \n _ _ _ _ 0.000803374171520386 S c r o o g e . 0.000803374171520386 a i d _ S c r o 0.000803374171520386 s a i d _ S c r 0.000803374171520386 S c r o o g e ' 0.000702952400080337 c r o o g e ' s 0.000702952400080337 _ n e p h e w . 0.000602530628640289 _ o f _ t h e _ 0.000602530628640289 ! ' \n \n _ _ _ _ 0.000602530628640289 e r r y _ C h r 0.000602530628640289 r r y _ C h r i 0.000602530628640289 r y _ C h r i s 0.000602530628640289 y _ C h r i s t 0.000602530628640289 \n S c r o o g e 0.000502108857200241 _ h i s _ s o l 0.000502108857200241 , ' _ s a i d _ 0.000502108857200241 S c r o o g e , 0.000502108857200241 c r o o g e , _ 0.000502108857200241 e p h e w . _ \` 0.000502108857200241 g h t _ h a v e 0.000502108857200241 h i s _ s o l e 0.000502108857200241 i g h t _ h a v 0.000502108857200241 n e p h e w . _ 0.000502108857200241 t h a t _ h e _ 0.000502108857200241 \n \n _ _ _ _ _ S 0.000401687085760193 \n _ _ _ _ _ S c 0.000401687085760193 \n _ _ _ _ _ T h 0.000401687085760193 \n _ _ _ _ _ \` B 0.000401687085760193 _ _ _ _ _ S c r 0.000401687085760193 _ _ _ _ _ T h e 0.000401687085760193 _ _ _ _ S c r o 0.000401687085760193 _ _ _ S c r o o 0.000401687085760193 _ _ S c r o o g 0.000401687085760193 _ I t _ w a s _ 0.000401687085760193 _ a _ t i m e _ 0.000401687085760193 _ h a v e _ y o 0.000401687085760193 _ h i s _ o w n 0.000401687085760193 _ i t _ h a s _ 0.000401687085760193 _ o n _ h i s _ 0.000401687085760193 _ r e t u r n e 0.000401687085760193 _ t h a t _ h e 0.000401687085760193 _ t h e _ n e p 0.000401687085760193 ' _ r e t u r n 0.000401687085760193 , _ h i s _ s o 0.000401687085760193 ? ' \n \n _ _ _ _ 0.000401687085760193 c r o o g e . \n 0.000401687085760193 c r o o g e . _ 0.000401687085760193 d _ M a r l e y 0.000401687085760193 d _ t h e _ n e 0.000401687085760193 e _ n e p h e w 0.000401687085760193 e . \n \n _ _ _ _ 0.000401687085760193 e d _ S c r o o 0.000401687085760193 e t u r n e d _ 0.000401687085760193 g _ t h e i r _ 0.000401687085760193 h a t _ h e _ w 0.000401687085760193 h a v e _ y o u 0.000401687085760193 h e _ c l e r k 0.000401687085760193 h e _ n e p h e 0.000401687085760193 h i s _ o w n _ 0.000401687085760193 h t _ h a v e _ 0.000401687085760193 i s _ s o l e _ 0.000401687085760193 n e d _ t h e _ 0.000401687085760193 n o t h i n g _ 0.000401687085760193 o u l d _ b e _ 0.000401687085760193 r e t u r n e d 0.000401687085760193 r i s t m a s _ 0.000401687085760193 r n e d _ t h e 0.000401687085760193 r o o g e ' s _ 0.000401687085760193 s _ n e p h e w 0.000401687085760193 t h e _ c l e r 0.000401687085760193 t h e _ n e p h 0.000401687085760193 t u r n e d _ t 0.000401687085760193 u r n e d _ t h 0.000401687085760193 \n \n _ _ _ _ _ O 0.000301265314320145 \n \n _ _ _ _ _ T 0.000301265314320145 _ _ _ _ T h e _ 0.000301265314320145 _ I _ h a v e _ 0.000301265314320145 _ M a r l e y ' 0.000301265314320145 _ Y o u ' r e _ 0.000301265314320145 _ \` G o o d _ a 0.000301265314320145 _ a f t e r n o 0.000301265314320145 _ d o o r - n a 0.000301265314320145 _ h e _ w a s _ 0.000301265314320145 _ h i m _ i n _ 0.000301265314320145 _ h i s _ e y e 0.000301265314320145 _ i n _ h i s _ 0.000301265314320145 9-GRAMS (total count: 9957) FIRST N-GRAM: A _ C H R I S T M LAST N-GRAM: S c r o o g e . \n --------------------------- _ _ _ _ _ _ _ _ _ 0.00321381942352114 _ S c r o o g e _ 0.00150647785477553 d _ S c r o o g e 0.00150647785477553 C h r i s t m a s 0.0014060459977905 ' \n \n _ _ _ _ _ \` 0.00130561414080546 _ C h r i s t m a 0.00120518228382043 . \n \n _ _ _ _ _ \` 0.000903886712865321 i d _ S c r o o g 0.000903886712865321 _ s a i d _ S c r 0.000803454855880285 ' _ s a i d _ S c 0.000803454855880285 . ' \n \n _ _ _ _ _ 0.000803454855880285 a i d _ S c r o o 0.000803454855880285 s a i d _ S c r o 0.000803454855880285 S c r o o g e ' s 0.00070302299889525 _ S c r o o g e ' 0.000602591141910214 _ S c r o o g e . 0.000602591141910214 ! ' \n \n _ _ _ _ _ 0.000602591141910214 e r r y _ C h r i 0.000602591141910214 r r y _ C h r i s 0.000602591141910214 r y _ C h r i s t 0.000602591141910214 y _ C h r i s t m 0.000602591141910214 _ h i s _ s o l e 0.000502159284925178 , ' _ s a i d _ S 0.000502159284925178 S c r o o g e , _ 0.000502159284925178 i g h t _ h a v e 0.000502159284925178 n e p h e w . _ \` 0.000502159284925178 \n \n _ _ _ _ _ S c 0.000401727427940143 \n \n _ _ _ _ _ \` B 0.000401727427940143 \n _ _ _ _ _ S c r 0.000401727427940143 \n _ _ _ _ _ T h e 0.000401727427940143 _ _ _ _ _ S c r o 0.000401727427940143 _ _ _ _ S c r o o 0.000401727427940143 _ _ _ S c r o o g 0.000401727427940143 _ _ S c r o o g e 0.000401727427940143 _ S c r o o g e , 0.000401727427940143 _ h a v e _ y o u 0.000401727427940143 _ h i s _ o w n _ 0.000401727427940143 _ n e p h e w . _ 0.000401727427940143 _ r e t u r n e d 0.000401727427940143 _ t h a t _ h e _ 0.000401727427940143 _ t h e _ n e p h 0.000401727427940143 ' _ r e t u r n e 0.000401727427940143 , _ h i s _ s o l 0.000401727427940143 ? ' \n \n _ _ _ _ _ 0.000401727427940143 S c r o o g e . \n 0.000401727427940143 S c r o o g e . _ 0.000401727427940143 c r o o g e ' s _ 0.000401727427940143 d _ t h e _ n e p 0.000401727427940143 e . \n \n _ _ _ _ _ 0.000401727427940143 e d _ S c r o o g 0.000401727427940143 e t u r n e d _ t 0.000401727427940143 g h t _ h a v e _ 0.000401727427940143 h e _ n e p h e w 0.000401727427940143 h i s _ s o l e _ 0.000401727427940143 h r i s t m a s _ 0.000401727427940143 r e t u r n e d _ 0.000401727427940143 r n e d _ t h e _ 0.000401727427940143 t h a t _ h e _ w 0.000401727427940143 t h e _ n e p h e 0.000401727427940143 t u r n e d _ t h 0.000401727427940143 u r n e d _ t h e 0.000401727427940143 \n \n _ _ _ _ _ T h 0.000301295570955107 _ _ _ _ _ T h e _ 0.000301295570955107 _ M a r l e y ' s 0.000301295570955107 _ \` G o o d _ a f 0.000301295570955107 _ a _ t i m e _ f 0.000301295570955107 _ a f t e r n o o 0.000301295570955107 _ d o o r - n a i 0.000301295570955107 _ m e r r y _ C h 0.000301295570955107 _ m i g h t _ h a 0.000301295570955107 _ n o t h i n g _ 0.000301295570955107 _ o f _ S c r o o 0.000301295570955107 _ t h e _ c l e r 0.000301295570955107 _ t i m e _ f o r 0.000301295570955107 , ' _ r e t u r n 0.000301295570955107 . \n \n _ _ _ _ _ O 0.000301295570955107 . _ I t _ w a s _ 0.000301295570955107 . _ S c r o o g e 0.000301295570955107 G o o d _ a f t e 0.000301295570955107 M a r l e y _ w a 0.000301295570955107 M a r l e y ' s _ 0.000301295570955107 M e r r y _ C h r 0.000301295570955107 S c r o o g e _ a 0.000301295570955107 S c r o o g e _ s 0.000301295570955107 \` G o o d _ a f t 0.000301295570955107 a _ d o o r - n a 0.000301295570955107 a _ t i m e _ f o 0.000301295570955107 a f t e r n o o n 0.000301295570955107 a r l e y _ w a s 0.000301295570955107 c r o o g e _ a n 0.000301295570955107 c r o o g e . \n \n 0.000301295570955107 d _ a f t e r n o 0.000301295570955107 d . \n \n _ _ _ _ _ 0.000301295570955107 d o o r - n a i l 0.000301295570955107 e _ n e p h e w . 0.000301295570955107 e d _ t h e _ n e 0.000301295570955107 f _ S c r o o g e 0.000301295570955107 g e . \n \n _ _ _ _ 0.000301295570955107 h r i s t m a s \n 0.000301295570955107 h r i s t m a s ! 0.000301295570955107 10-GRAMS (total count: 9956) FIRST N-GRAM: A _ C H R I S T M A LAST N-GRAM: _ S c r o o g e . \n ---------------------------- _ _ _ _ _ _ _ _ _ _ 0.00311370028123744 _ C h r i s t m a s 0.00120530333467256 i d _ S c r o o g e 0.000903977501004419 _ s a i d _ S c r o 0.000803535556448373 ' _ s a i d _ S c r 0.000803535556448373 a i d _ S c r o o g 0.000803535556448373 s a i d _ S c r o o 0.000803535556448373 _ S c r o o g e ' s 0.00060265166733628 . ' \n \n _ _ _ _ _ \` 0.00060265166733628 e r r y _ C h r i s 0.00060265166733628 r r y _ C h r i s t 0.00060265166733628 r y _ C h r i s t m 0.00060265166733628 y _ C h r i s t m a 0.00060265166733628 , ' _ s a i d _ S c 0.000502209722780233 d _ S c r o o g e _ 0.000502209722780233 d _ S c r o o g e . 0.000502209722780233 \n \n _ _ _ _ _ S c r 0.000401767778224186 \n _ _ _ _ _ S c r o 0.000401767778224186 _ _ _ _ _ S c r o o 0.000401767778224186 _ _ _ _ S c r o o g 0.000401767778224186 _ _ _ S c r o o g e 0.000401767778224186 _ _ S c r o o g e _ 0.000401767778224186 _ S c r o o g e , _ 0.000401767778224186 _ S c r o o g e . \n 0.000401767778224186 _ h i s _ s o l e _ 0.000401767778224186 _ n e p h e w . _ \` 0.000401767778224186 _ r e t u r n e d _ 0.000401767778224186 _ t h a t _ h e _ w 0.000401767778224186 _ t h e _ n e p h e 0.000401767778224186 ' _ r e t u r n e d 0.000401767778224186 , _ h i s _ s o l e 0.000401767778224186 ? ' \n \n _ _ _ _ _ \` 0.000401767778224186 C h r i s t m a s _ 0.000401767778224186 S c r o o g e ' s _ 0.000401767778224186 d _ t h e _ n e p h 0.000401767778224186 e d _ S c r o o g e 0.000401767778224186 e t u r n e d _ t h 0.000401767778224186 i g h t _ h a v e _ 0.000401767778224186 r e t u r n e d _ t 0.000401767778224186 t h e _ n e p h e w 0.000401767778224186 t u r n e d _ t h e 0.000401767778224186 u r n e d _ t h e _ 0.000401767778224186 \n \n _ _ _ _ _ T h e 0.00030132583366814 \n _ _ _ _ _ T h e _ 0.00030132583366814 _ M a r l e y ' s _ 0.00030132583366814 _ S c r o o g e _ s 0.00030132583366814 _ \` G o o d _ a f t 0.00030132583366814 _ a _ t i m e _ f o 0.00030132583366814 _ a f t e r n o o n 0.00030132583366814 _ d o o r - n a i l 0.00030132583366814 _ m e r r y _ C h r 0.00030132583366814 _ m i g h t _ h a v 0.00030132583366814 _ o f _ S c r o o g 0.00030132583366814 _ t i m e _ f o r _ 0.00030132583366814 ! ' \n \n _ _ _ _ _ \` 0.00030132583366814 , ' _ r e t u r n e 0.00030132583366814 . \n \n _ _ _ _ _ \` B 0.00030132583366814 C h r i s t m a s \n 0.00030132583366814 C h r i s t m a s ! 0.00030132583366814 G o o d _ a f t e r 0.00030132583366814 M a r l e y _ w a s 0.00030132583366814 M e r r y _ C h r i 0.00030132583366814 S c r o o g e _ a n 0.00030132583366814 S c r o o g e . \n \n 0.00030132583366814 \` G o o d _ a f t e 0.00030132583366814 a _ d o o r - n a i 0.00030132583366814 a _ t i m e _ f o r 0.00030132583366814 a r l e y _ w a s _ 0.00030132583366814 c r o o g e _ a n d 0.00030132583366814 c r o o g e ' s _ n 0.00030132583366814 c r o o g e . \n \n _ 0.00030132583366814 d _ S c r o o g e ' 0.00030132583366814 d _ a f t e r n o o 0.00030132583366814 d o o r - n a i l . 0.00030132583366814 e d _ t h e _ n e p 0.00030132583366814 f _ S c r o o g e ' 0.00030132583366814 g e . \n \n _ _ _ _ _ 0.00030132583366814 h e _ n e p h e w . 0.00030132583366814 m e r r y _ C h r i 0.00030132583366814 m i g h t _ h a v e 0.00030132583366814 n , ' _ s a i d _ S 0.00030132583366814 o d _ a f t e r n o 0.00030132583366814 o f _ S c r o o g e 0.00030132583366814 o g e . \n \n _ _ _ _ 0.00030132583366814 o o d _ a f t e r n 0.00030132583366814 o o g e . \n \n _ _ _ 0.00030132583366814 r o o g e _ a n d _ 0.00030132583366814 r o o g e . \n \n _ _ 0.00030132583366814 s _ n e p h e w . _ 0.00030132583366814 \n \n _ _ _ _ _ \` B e 0.000200883889112093 \n \n _ _ _ _ _ \` D o 0.000200883889112093 \n \n _ _ _ _ _ \` G o 0.000200883889112093 \n \n _ _ _ _ _ \` I _ 0.000200883889112093 \n \n _ _ _ _ _ \` L e 0.000200883889112093 \n \n _ _ _ _ _ \` W h 0.000200883889112093 \n _ _ _ _ _ \` B e c 0.000200883889112093 \n _ _ _ _ _ \` D o n 0.000200883889112093 \n _ _ _ _ _ \` G o o 0.000200883889112093 \n _ _ _ _ _ \` L e t 0.000200883889112093 \n S c r o o g e . _ 0.000200883889112093 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/11.t0000644000175000017500000000051312110171565013534 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 1; require 't/auxfunctions.pl'; my $com = "perl -Mblib ./ngrams.pl --n=10 --orderby=frequency --type=byte". " --normalize --onlyfirst=100 t/11.in"; my $out = scalar(`$com`); #putfile('t/11.out-new', $out); is(normalize(scalar(getfile('t/11.out'))), normalize($out)); Text-Ngrams-2.005/t/12.t0000644000175000017500000000044712110172164013537 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 1; require 't/auxfunctions.pl'; my $com = "perl -Mblib ./ngrams.pl --n=10 --orderby=frequency --type=character". " --normalize --onlyfirst=100 t/12.in"; my $out = scalar(`$com`); #putfile('t/12.out-new', $out); &isn('t/12.out', $out); Text-Ngrams-2.005/t/05-1.out0000644000175000017500000000077210350310204014235 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 1.8 1-GRAMS (total count: 22) FIRST N-GRAM: M LAST N-GRAM: \n ------------------------- \n 11 M 1 i 4 p 2 s 4 2-GRAMS (total count: 21) FIRST N-GRAM: M \n LAST N-GRAM: i \n ------------------------- \n i 4 \n p 2 \n s 4 M \n 1 i \n 4 p \n 2 s \n 4 3-GRAMS (total count: 20) FIRST N-GRAM: M \n i LAST N-GRAM: \n i \n ------------------------- \n i \n 4 \n p \n 2 \n s \n 4 M \n i 1 i \n p 1 i \n s 2 p \n i 1 p \n p 1 s \n i 2 s \n s 2 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/05-0.in0000644000175000017500000000001410350310204014020 0ustar vladovladomississippi Text-Ngrams-2.005/t/06.t0000644000175000017500000000033111100610551013525 0ustar vladovlado#!/usr/bin/perl use Test::More tests => 1; require 't/auxfunctions.pl'; my $out = `$^X -Mblib ./ngrams.pl --n=2 --orderby=ngram --type=word t/05.in`; is(normalize(scalar(getfile('t/03.out'))), normalize($out)); Text-Ngrams-2.005/t/14.in0000644000175000017500000000007211741077615013714 0ustar vladovladofirst line unix second line dos third line mac last line Text-Ngrams-2.005/t/10.out0000644000175000017500000000004607770361403014110 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version Text-Ngrams-2.005/t/01.out0000644000175000017500000000070410350307337014103 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 1.8 1-GRAMS (total count: 8) FIRST N-GRAM: a LAST N-GRAM: h ------------------------ a 1 b 1 c 1 d 1 e 1 f 1 g 1 h 1 2-GRAMS (total count: 7) FIRST N-GRAM: a b LAST N-GRAM: g h ------------------------ a b 1 b c 1 c d 1 d e 1 e f 1 f g 1 g h 1 3-GRAMS (total count: 6) FIRST N-GRAM: a b c LAST N-GRAM: f g h ------------------------ a b c 1 b c d 1 c d e 1 d e f 1 e f g 1 f g h 1 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/04.out0000644000175000017500000000155610350307337014114 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 1-GRAMS (total count: 55) FIRST N-GRAM: \t LAST N-GRAM: \n ------------------------- \t 3 \n 3 _ 12 , 2 . 3 T 1 b 3 c 1 e 1 f 3 h 1 i 1 k 1 n 3 o 6 q 1 r 3 u 1 w 3 x 3 2-GRAMS (total count: 54) FIRST N-GRAM: \t T LAST N-GRAM: . \n ------------------------- \t _ 1 \t T 1 \t b 1 \n \t 2 _ _ 5 _ . 1 _ b 2 _ f 3 _ q 1 , \n 2 . \n 1 . . 2 T h 1 b r 3 c k 1 e _ 1 f o 3 h e 1 i c 1 k _ 1 n _ 3 o w 3 o x 3 q u 1 r o 3 u i 1 w n 3 x _ 1 x , 2 3-GRAMS (total count: 53) FIRST N-GRAM: \t T h LAST N-GRAM: . . \n ------------------------- \t _ _ 1 \t T h 1 \t b r 1 \n \t _ 1 \n \t b 1 _ _ _ 4 _ _ b 1 _ . . 1 _ b r 2 _ f o 3 _ q u 1 , \n \t 2 . . \n 1 . . . 1 T h e 1 b r o 3 c k _ 1 e _ b 1 f o x 3 h e _ 1 i c k 1 k _ f 1 n _ f 2 n _ q 1 o w n 3 o x _ 1 o x , 2 q u i 1 r o w 3 u i c 1 w n _ 3 x _ . 1 x , \n 2 END OUTPUT BY Text::Ngrams Text-Ngrams-2.005/t/02.out0000644000175000017500000000115410350307337014104 0ustar vladovladoBEGIN OUTPUT BY Text::Ngrams version 1.8 1-GRAMS (total count: 17) FIRST N-GRAM: A LAST N-GRAM: P ------------------------- _ 1 A 1 B 1 C 1 D 1 E 1 F 1 G 1 H 1 I 1 J 1 K 1 L 1 M 1 N 1 O 1 P 1 2-GRAMS (total count: 16) FIRST N-GRAM: A B LAST N-GRAM: O P ------------------------- _ H 1 A B 1 B C 1 C D 1 D E 1 E F 1 F G 1 G _ 1 H I 1 I J 1 J K 1 K L 1 L M 1 M N 1 N O 1 O P 1 3-GRAMS (total count: 15) FIRST N-GRAM: A B C LAST N-GRAM: N O P ------------------------- _ H I 1 A B C 1 B C D 1 C D E 1 D E F 1 E F G 1 F G _ 1 G _ H 1 H I J 1 I J K 1 J K L 1 K L M 1 L M N 1 M N O 1 N O P 1 END OUTPUT BY Text::Ngrams