tag (linking to the exact search) is wrapped in the
# span.spell_orig.
def __init__(self, news=False):
html.parser.HTMLParser.__init__(self)
self.news = news
self.autocorrected = False
self.suggested_spelling = None
self.filtered = False
self.results = []
self.index = 0
self.textbuf = ''
self.tag_annotations = {}
self.set_handlers_to('root')
# Ignore list
IGNORE_LIST = [
# "People also ask"
# Sample query: VPN
# Screenshot: https://i.imgur.com/yfcsoQz.png
{
'tag': 'div',
'class': 'related-question-pair'
},
# We omit Google's "smart card" results (term coined by me) by
# guarding against the 'g-blk' class (sample response: https://git.io/voJgB)
{
'tag': 'div',
'class': 'g-blk'
},
# We also guard against "smart-card" results with `--noua` option
{
'tag': 'div',
'class': 'hp-xpdbox'
}
]
# Tag handlers
@annotate_tag
def root_start(self, tag, attrs):
if tag == 'div' and 'g' in self.classes(attrs):
# Initialize result field registers
self.title = ''
self.url = ''
self.abstract = ''
self.metadata = '' # Only used for Google News
self.sitelinks = []
# Guard against sitelinks, which also have titles and
# abstracts. In the case of news, guard against "card
# sections" (secondary results to the same event).
self.title_registered = False
self.abstract_registered = False
self.metadata_registered = False # Only used for Google News
self.set_handlers_to('result')
return 'result'
# Autocorrect
if tag == 'span' and 'spell_orig' in self.classes(attrs) and attrs.get('id') != 'sifm':
self.autocorrected = True
return
if tag == 'a' and 'spell' in self.classes(attrs) and attrs.get('id') != 'srfl':
self.start_populating_textbuf()
return 'spell'
# Omitted results
if tag == 'p' and attrs.get('id') == 'ofr':
self.filtered = True
@retrieve_tag_annotation
def root_end(self, tag, annotation):
if annotation == 'spell':
self.stop_populating_textbuf()
self.suggested_spelling = self.pop_textbuf()
@annotate_tag
def result_start(self, tag, attrs):
if not ua and tag == 'span' and 'mime' in self.classes(attrs):
self.start_populating_textbuf()
return 'title_filetype'
if not self.title_registered and tag == 'h3' and 'r' in self.classes(attrs):
self.set_handlers_to('title')
return 'title'
if not self.abstract_registered and tag == 'div' and 's' in self.classes(attrs):
self.set_handlers_to('abstract')
return 'abstract'
if not ua and not self.abstract_registered \
and tag == 'span' and 'st' in self.classes(attrs):
self.start_populating_textbuf(lambda text: text + ' ')
return 'abstract_gservices'
if not self.sitelinks and tag == 'table':
if ua or (not self.news and 'ts' not in self.classes(attrs)):
self.set_handlers_to('sitelink_table')
return 'sitelink_table'
if self.news:
if not self.metadata_registered and tag == 'div' and 'slp' in self.classes(attrs):
# Change metadata field separator from '-' to ', ' for better appearance
if ua:
self.start_populating_textbuf(lambda text: ', ' if text == '-' else text)
else:
self.start_populating_textbuf(lambda text:
text.replace(' -', ',', 1) if ' - ' in text else text)
return 'news_metadata'
if not self.abstract_registered and tag == 'div' and 'st' in self.classes(attrs):
self.start_populating_textbuf()
return 'news_abstract'
@retrieve_tag_annotation
def result_end(self, tag, annotation):
if annotation == 'result':
if self.url:
self.index += 1
result = Result(self.index, self.title, self.url, self.abstract,
metadata=self.metadata if self.metadata else None,
sitelinks=self.sitelinks)
self.results.append(result)
self.set_handlers_to('root')
elif annotation == 'news_metadata':
self.stop_populating_textbuf()
self.metadata = self.pop_textbuf()
self.metadata_registered = True
elif annotation == 'news_abstract':
self.stop_populating_textbuf()
self.abstract = self.pop_textbuf()
self.abstract_registered = True
elif annotation == 'abstract_gservices':
self.stop_populating_textbuf()
self.abstract = self.pop_textbuf().replace(' ', ' ')
self.abstract_registered = False
@annotate_tag
def title_start(self, tag, attrs):
if ua and tag == 'span':
# Print a space after the filetype indicator
self.start_populating_textbuf(lambda text: text + ' ')
return 'title_filetype'
if tag == 'a' and 'href' in attrs:
# Skip 'News for', 'Images for' search links
if attrs['href'].startswith('/search'):
return
self.url = attrs['href']
try:
start = self.url.index('?q=') + len('?q=')
end = self.url.index('&sa=', start)
self.url = urllib.parse.unquote_plus(self.url[start:end])
except ValueError:
pass
self.start_populating_textbuf()
return 'title_link'
@retrieve_tag_annotation
def title_end(self, tag, annotation):
if annotation == 'title_filetype':
self.stop_populating_textbuf()
elif annotation == 'title_link':
self.stop_populating_textbuf()
self.title = self.pop_textbuf()
self.title_registered = True
elif annotation == 'title':
self.set_handlers_to('result')
@annotate_tag
def abstract_start(self, tag, attrs):
if tag == 'span' and 'st' in self.classes(attrs):
self.start_populating_textbuf()
return 'abstract_text'
@retrieve_tag_annotation
def abstract_end(self, tag, annotation):
if annotation == 'abstract_text':
self.stop_populating_textbuf()
self.abstract = self.pop_textbuf()
self.abstract_registered = False
elif annotation == 'abstract':
self.set_handlers_to('result')
@annotate_tag
def sitelink_table_start(self, tag, attrs):
if tag == 'td':
# Initialize a new sitelink
self.current_sitelink = Sitelink('', '', '')
self.set_handlers_to('sitelink')
return 'sitelink'
@retrieve_tag_annotation
def sitelink_table_end(self, tag, annotation):
if annotation == 'sitelink_table':
self.set_handlers_to('result')
@annotate_tag
def sitelink_start(self, tag, attrs):
if tag == 'h3' and 'r' in self.classes(attrs):
self.set_handlers_to('sitelink_title')
return 'sitelink_title'
if tag == 'div' and 'st' in self.classes(attrs):
self.start_populating_textbuf()
return 'sitelink_abstract'
@retrieve_tag_annotation
def sitelink_end(self, tag, annotation):
if annotation == 'sitelink_abstract':
self.stop_populating_textbuf()
self.current_sitelink.abstract = self.pop_textbuf()
elif annotation == 'sitelink':
if self.current_sitelink.url:
self.sitelinks.append(self.current_sitelink)
self.set_handlers_to('sitelink_table')
@annotate_tag
def sitelink_title_start(self, tag, attrs):
if tag == 'a' and 'href' in attrs:
self.current_sitelink.url = attrs['href']
try:
start = self.current_sitelink.url.index('?q=') + len('?q=')
end = self.current_sitelink.url.index('&sa=', start)
self.current_sitelink.url = urllib.parse.unquote_plus(self.current_sitelink.url[start:end])
except ValueError:
pass
self.start_populating_textbuf()
return 'sitelink_title_link'
@retrieve_tag_annotation
def sitelink_title_end(self, tag, annotation):
if annotation == 'sitelink_title_link':
self.stop_populating_textbuf()
self.current_sitelink.title = self.pop_textbuf()
elif annotation == 'sitelink_title':
self.set_handlers_to('sitelink')
# Generic methods
# Set handle_starttag to SCOPE_start, and handle_endtag to SCOPE_end.
def set_handlers_to(self, scope):
self.handle_starttag = getattr(self, scope + '_start')
self.handle_endtag = getattr(self, scope + '_end')
def insert_annotation(self, tag, annotation):
if tag not in self.tag_annotations:
self.tag_annotations[tag] = []
self.tag_annotations[tag].append(annotation)
@annotate_tag
def ignore_start(self, tag, attrs):
pass
@retrieve_tag_annotation
def ignore_end(self, tag, annotation):
if annotation and annotation.endswith('_ignored'):
# Strip '-ignore' suffix from annotation to obtain the outer
# context name.
context = annotation[:-8]
self.set_handlers_to(context)
def start_populating_textbuf(self, data_transformer=None):
if data_transformer is None:
# Record data verbatim
self.handle_data = self.record_data
else:
def record_transformed_data(data):
self.textbuf += data_transformer(data)
self.handle_data = record_transformed_data
self.handle_entityref = self.record_entityref
self.handle_charref = self.record_charref
def pop_textbuf(self):
text = self.textbuf
self.textbuf = ''
return text
def stop_populating_textbuf(self):
self.handle_data = lambda data: None
self.handle_entityref = lambda ref: None
self.handle_charref = lambda ref: None
def record_data(self, data):
self.textbuf += data
def record_entityref(self, ref):
try:
self.textbuf += chr(html.entities.name2codepoint[ref])
except KeyError:
# Entity name not found; most likely rather sloppy HTML
# where a literal ampersand is not escaped; For instance,
# the HTML response returned by
#
# googler -c au -l ko expected
#
# contains the following tag
#
# expected market return s&p 500
#
# where &p is interpreted by HTMLParser as an entity (this
# behaviour seems to be specific to Python 2.7).
self.textbuf += '&' + ref
def record_charref(self, ref):
if ref.startswith('x'):
char = chr(int(ref[1:], 16))
else:
char = chr(int(ref))
self.textbuf += char
@staticmethod
def classes(attrs):
"""Get tag's classes from its attribute dict."""
return attrs.get('class', '').split()
class Sitelink(object):
"""Container for a sitelink."""
def __init__(self, title, url, abstract):
self.title = title
self.url = url
self.abstract = abstract
self.index = ''
Colors = collections.namedtuple('Colors', 'index, title, url, metadata, abstract, prompt, reset')
class Result(object):
"""
Container for one search result, with output helpers.
Parameters
----------
index : int or str
title : str
url : str
abstract : str
metadata : str, optional
Only applicable to Google News results, with publisher name and
publishing time.
sitelinks : list, optional
List of ``SiteLink`` objects.
Attributes
----------
index : str
title : str
url : str
abstract : str
metadata : str or None
sitelinks : list
Class Variables
---------------
colors : str
Methods
-------
print()
jsonizable_object()
urltable()
"""
# Class variables
colors = None
def __init__(self, index, title, url, abstract, metadata=None, sitelinks=None):
index = str(index)
self.index = index
self.title = title
self.url = url
self.abstract = abstract
self.metadata = metadata
self.sitelinks = [] if sitelinks is None else sitelinks
self._urltable = {index: url}
subindex = 'a'
for sitelink in sitelinks:
fullindex = index + subindex
sitelink.index = fullindex
self._urltable[fullindex] = sitelink.url
subindex = chr(ord(subindex) + 1)
def _print_title_and_url(self, index, title, url, indent=0):
colors = self.colors
# Pad index and url with `indent` number of spaces
index = ' ' * indent + str(index)
url = ' ' * indent + url
if colors:
print(colors.index + index + colors.reset, end='')
print(' ' + colors.title + title + colors.reset)
print(colors.url + url + colors.reset)
else:
print(' %s %s\n%s' % (index, title, url))
def _print_metadata_and_abstract(self, abstract, metadata=None, indent=0):
colors = self.colors
try:
columns, _ = os.get_terminal_size()
except OSError:
columns = 0
if metadata:
if colors:
print(colors.metadata + metadata + colors.reset)
else:
print(metadata)
if colors:
print(colors.abstract, end='')
if columns > indent + 1:
# Try to fill to columns
fillwidth = columns - indent - 1
for line in textwrap.wrap(abstract.replace('\n', ''), width=fillwidth):
print('%s%s' % (' ' * indent, line))
print('')
else:
print('%s\n' % abstract.replace('\n', ' '))
if colors:
print(colors.reset, end='')
def print(self):
"""Print the result entry."""
self._print_title_and_url(self.index, self.title, self.url)
self._print_metadata_and_abstract(self.abstract, metadata=self.metadata)
for sitelink in self.sitelinks:
self._print_title_and_url(sitelink.index, sitelink.title, sitelink.url, indent=4)
self._print_metadata_and_abstract(sitelink.abstract, indent=4)
def jsonizable_object(self):
"""Return a JSON-serializable dict representing the result entry."""
obj = {
'title': self.title,
'url': self.url,
'abstract': self.abstract
}
if self.metadata:
obj['metadata'] = self.metadata
if self.sitelinks:
obj['sitelinks'] = [sitelink.__dict__ for sitelink in self.sitelinks]
return obj
def urltable(self):
"""Return a index-to-URL table for the current result.
Normally, the table contains only a single entry, but when the result
contains sitelinks, all sitelinks are included in this table.
Returns
-------
dict
A dict mapping indices (strs) to URLs (also strs). Indices of
sitelinks are the original index appended by lowercase letters a,
b, c, etc.
"""
return self._urltable
class GooglerCmdException(Exception):
pass
class NoKeywordsException(GooglerCmdException):
pass
def require_keywords(method):
# Require keywords to be set before we run a GooglerCmd method. If
# no keywords have been set, raise a NoKeywordsException.
@functools.wraps(method)
def enforced_method(self, *args, **kwargs):
if not self.keywords:
raise NoKeywordsException('No keywords.')
method(self, *args, **kwargs)
return enforced_method
def no_argument(method):
# Normalize a do_* method of GooglerCmd that takes no argument to
# one that takes an arg, but issue a warning when an nonempty
# argument is given.
@functools.wraps(method)
def enforced_method(self, arg):
if arg:
method_name = arg.__name__
command_name = method_name[3:] if method_name.startswith('do_') else method_name
logger.warning("Argument to the '%s' command ignored.", command_name)
method(self)
return enforced_method
class GooglerCmd(object):
"""
Command line interpreter and executor class for googler.
Inspired by PSL cmd.Cmd.
Parameters
----------
opts : argparse.Namespace
Options and/or arguments.
Attributes
----------
options : argparse.Namespace
Options that are currently in effect. Read-only attribute.
keywords : str or list or strs
Current keywords. Read-only attribute
Methods
-------
fetch()
display_results(prelude='\n', json_output=False)
fetch_and_display(prelude='\n', json_output=False, interactive=True)
read_next_command()
help()
cmdloop()
"""
# Class variables
colors = None
def __init__(self, opts):
super().__init__()
self._opts = opts
self._google_url = GoogleUrl(opts)
proxy = opts.proxy if hasattr(opts, 'proxy') else None
self._conn = GoogleConnection(self._google_url.hostname, proxy=proxy,
notweak=opts.notweak)
atexit.register(self._conn.close)
self.results = []
self._autocorrected_to = None
self._results_filtered = False
self._urltable = {}
@property
def options(self):
"""Current options."""
return self._opts
@property
def keywords(self):
"""Current keywords."""
return self._google_url.keywords
@require_keywords
def fetch(self):
"""Fetch a page and parse for results.
Results are stored in ``self.results``.
Raises
------
GoogleConnectionError
See Also
--------
fetch_and_display
"""
# This method also sets self._results_filtered and
# self._urltable.
page = self._conn.fetch_page(self._google_url.relative())
if logger.isEnabledFor(logging.DEBUG):
import tempfile
fd, tmpfile = tempfile.mkstemp(prefix='googler-response-')
os.close(fd)
with open(tmpfile, 'w', encoding='utf-8') as fp:
fp.write(page)
logger.debug("Response body written to '%s'.", tmpfile)
parser = GoogleParser(news=self._google_url.news)
parser.feed(page)
self.results = parser.results
self._autocorrected_to = parser.suggested_spelling if parser.autocorrected else None
self._results_filtered = parser.filtered
self._urltable = {}
for r in self.results:
self._urltable.update(r.urltable())
@require_keywords
def display_results(self, prelude='\n', json_output=False):
"""Display results stored in ``self.results``.
Parameters
----------
See `fetch_and_display`.
"""
if json_output:
# JSON output
import json
results_object = [r.jsonizable_object() for r in self.results]
print(json.dumps(results_object, indent=2, sort_keys=True, ensure_ascii=False))
else:
# Regular output
if not self.results:
print('No results.', file=sys.stderr)
else:
sys.stderr.write(prelude)
for r in self.results:
r.print()
@require_keywords
def fetch_and_display(self, prelude='\n', json_output=False, interactive=True):
"""Fetch a page and display results.
Results are stored in ``self.results``.
Parameters
----------
prelude : str, optional
A string that is written to stderr before showing actual results,
usually serving as a separator. Default is an empty line.
json_output : bool, optional
Whether to dump results in JSON format. Default is False.
interactive : bool, optional
Whether to show contextual instructions, when e.g. Google
has filtered the results. Default is True.
Raises
------
GoogleConnectionError
See Also
--------
fetch
display_results
"""
self.fetch()
colors = self.colors
if self._autocorrected_to:
if colors:
# Underline the keywords
autocorrected_to = '\x1b[4m' + self._autocorrected_to + '\x1b[24m'
else:
autocorrected_to = self._autocorrected_to
autocorrect_info = ('Showing results for %s; enter "x" for an exact search.' %
autocorrected_to)
printerr('')
if colors:
printerr(colors.prompt + autocorrect_info + colors.reset)
else:
printerr('** ' + autocorrect_info)
self.display_results(prelude=prelude, json_output=json_output)
if self._results_filtered:
unfilter_info = 'Enter "unfilter" to show similar results Google omitted.'
if colors:
printerr(colors.prompt + unfilter_info + colors.reset)
else:
printerr('** ' + unfilter_info)
printerr('')
def read_next_command(self):
"""Show omniprompt and read user command line.
Command line is always stripped, and each consecutive group of
whitespace is replaced with a single space character. If the
command line is empty after stripping, when ignore it and keep
reading. Exit with status 0 if we get EOF or an empty line
(pre-strip, that is, a raw ) twice in a row.
The new command line (non-empty) is stored in ``self.cmd``.
"""
colors = self.colors
message = 'googler (? for help)'
prompt = (colors.prompt + message + colors.reset + ' ') if colors else (message + ': ')
enter_count = 0
while True:
try:
cmd = input(prompt)
except EOFError:
sys.exit(0)
if not cmd:
enter_count += 1
if enter_count == 2:
# Double
sys.exit(0)
else:
enter_count = 0
cmd = ' '.join(cmd.split())
if cmd:
self.cmd = cmd
break
@staticmethod
def help():
GooglerArgumentParser.print_omniprompt_help(sys.stderr)
printerr('')
@require_keywords
@no_argument
def do_first(self):
self._google_url.first_page()
self.fetch_and_display()
def do_google(self, arg):
# Update keywords and reconstruct URL
self._opts.keywords = arg
self._google_url = GoogleUrl(self._opts)
self.fetch_and_display()
@require_keywords
@no_argument
def do_next(self):
# If > 5 results are being fetched each time,
# block next when no parsed results in current fetch
if not self.results and self._google_url._num > 5:
printerr('No results.')
else:
self._google_url.next_page()
self.fetch_and_display()
@require_keywords
def do_open(self, *args):
if not args:
open_url(self._google_url.full())
return
for nav in args:
if nav == 'a':
for key, value in sorted(self._urltable.items()):
open_url(self._urltable[key])
elif nav in self._urltable:
open_url(self._urltable[nav])
else:
printerr("Invalid index %s." % nav)
@require_keywords
@no_argument
def do_previous(self):
try:
self._google_url.prev_page()
except ValueError as e:
print(e, file=sys.stderr)
return
self.fetch_and_display()
@require_keywords
@no_argument
def do_exact(self):
# Reset start to 0 when exact is applied.
self._google_url.update(start=0, exact=True)
self.fetch_and_display()
@require_keywords
@no_argument
def do_unfilter(self):
# Reset start to 0 when unfilter is applied.
self._google_url.update(start=0)
self._google_url.set_queries(filter=0)
self.fetch_and_display()
def cmdloop(self):
"""Run REPL."""
if self.keywords:
self.fetch_and_display()
else:
printerr('Please initiate a query.')
while True:
self.read_next_command()
# TODO: Automatic dispatcher
#
# We can't write a dispatcher for now because that could
# change behaviour of the prompt. However, we have already
# laid a lot of ground work for the dispatcher, e.g., the
# `no_argument' decorator.
try:
cmd = self.cmd
if cmd == 'f':
self.do_first('')
elif cmd.startswith('g '):
self.do_google(cmd[2:])
elif cmd == 'n':
self.do_next('')
elif cmd == 'o':
self.do_open()
elif cmd.startswith('o '):
self.do_open(*cmd[2:].split())
elif cmd == 'p':
self.do_previous('')
elif cmd == 'q':
break
elif cmd == 'x':
self.do_exact('')
elif cmd == 'unfilter':
self.do_unfilter('')
elif cmd == '?':
self.help()
elif cmd in self._urltable:
open_url(self._urltable[cmd])
elif self.keywords and cmd.isdigit() and int(cmd) < 100:
printerr('Index out of bound. To search for the number, use g.')
else:
self.do_google(cmd)
except NoKeywordsException:
printerr('Initiate a query first.')
class GooglerArgumentParser(argparse.ArgumentParser):
"""Custom argument parser for googler."""
# Print omniprompt help
@staticmethod
def print_omniprompt_help(file=None):
file = sys.stderr if file is None else file
file.write(textwrap.dedent("""
omniprompt keys:
n, p fetch the next or previous set of search results
index open the result corresponding to index in browser
f jump to the first page
o [index ...] [a] open space-separated result indices, or all, in browser
open the current search in browser, if no arguments
g keywords new Google search for 'keywords' with original options
should be used to search omniprompt keys and indices
q, ^D, double Enter exit googler
? show omniprompt help
* other inputs issue a new search with original options
"""))
# Print information on googler
@staticmethod
def print_general_info(file=None):
file = sys.stderr if file is None else file
file.write(textwrap.dedent("""
Version %s
Copyright © 2008 Henri Hakkinen
Copyright © 2015-2017 Arun Prakash Jana
Zhiming Wang
License: GPLv3
Webpage: https://github.com/jarun/googler
""" % _VERSION_))
# Augment print_help to print more than synopsis and options
def print_help(self, file=None):
super().print_help(file)
self.print_omniprompt_help(file)
self.print_general_info(file)
# Automatically print full help text on error
def error(self, message):
sys.stderr.write('%s: error: %s\n\n' % (self.prog, message))
self.print_help(sys.stderr)
self.exit(2)
# Type guards
@staticmethod
def positive_int(arg):
"""Try to convert a string into a positive integer."""
try:
n = int(arg)
assert n > 0
return n
except (ValueError, AssertionError):
raise argparse.ArgumentTypeError('%s is not a positive integer' % arg)
@staticmethod
def nonnegative_int(arg):
"""Try to convert a string into a nonnegative integer."""
try:
n = int(arg)
assert n >= 0
return n
except (ValueError, AssertionError):
raise argparse.ArgumentTypeError('%s is not a non-negative integer' % arg)
@staticmethod
def is_duration(arg):
"""Check if a string is a valid duration accepted by Google.
A valid duration is of the form dNUM, where d is a single letter h
(hour), d (day), w (week), m (month), or y (year), and NUM is a
non-negative integer.
"""
try:
if arg[0] not in ('h', 'd', 'w', 'm', 'y') or int(arg[1:]) < 0:
raise ValueError
except (TypeError, IndexError, ValueError):
raise argparse.ArgumentTypeError('%s is not a valid duration' % arg)
return arg
@staticmethod
def is_colorstr(arg):
"""Check if a string is a valid color string."""
try:
assert len(arg) == 6
for c in arg:
assert c in COLORMAP
except AssertionError:
raise argparse.ArgumentTypeError('%s is not a valid color string' % arg)
return arg
# Self-upgrade mechanism
def system_is_windows():
"""Checks if the underlying system is Windows (Cygwin included)."""
return sys.platform in {'win32', 'cygwin'}
def download_latest_googler(include_git=False):
"""Download latest googler to a temp file.
By default, the latest released version is downloaded, but if
`include_git` is specified, then the latest git master is downloaded
instead.
Parameters
----------
include_git : bool, optional
Download from git master. Default is False.
Returns
-------
(git_ref, path): tuple
A tuple containing the git reference (either name of the latest
tag or SHA of the latest commit) and path to the downloaded
file.
"""
import urllib.request
if include_git:
# Get SHA of latest commit on master
request = urllib.request.Request('%s/commits/master' % API_REPO_BASE,
headers={'Accept': 'application/vnd.github.v3.sha'})
response = urllib.request.urlopen(request)
if response.status != 200:
raise http.client.HTTPException(response.reason)
git_ref = response.read().decode('utf-8')
else:
# Get name of latest tag
request = urllib.request.Request('%s/releases?per_page=1' % API_REPO_BASE,
headers={'Accept': 'application/vnd.github.v3+json'})
response = urllib.request.urlopen(request)
if response.status != 200:
raise http.client.HTTPException(response.reason)
import json
git_ref = json.loads(response.read().decode('utf-8'))[0]['tag_name']
# Download googler to a tempfile
googler_download_url = '%s/%s/googler' % (RAW_DOWNLOAD_REPO_BASE, git_ref)
printerr('Downloading %s' % googler_download_url)
request = urllib.request.Request(googler_download_url,
headers={'Accept-Encoding': 'gzip'})
import tempfile
fd, path = tempfile.mkstemp()
atexit.register(lambda: os.remove(path) if os.path.exists(path) else None)
os.close(fd)
with open(path, 'wb') as fp:
with urllib.request.urlopen(request) as response:
if response.status != 200:
raise http.client.HTTPException(response.reason)
payload = response.read()
try:
fp.write(gzip.decompress(payload))
except OSError:
fp.write(payload)
return git_ref, path
def self_replace(path):
"""Replace the current script with a specified file.
Both paths (the specified path and path to the current script) are
resolved to absolute, symlink-free paths. Upon replacement, the
owner and mode signatures of the current script are preserved. The
caller needs to have the necessary permissions.
Replacement won't happen if the specified file is the same
(content-wise) as the current script.
Parameters
----------
path : str
Path to the replacement file.
Returns
-------
bool
True if replaced, False if skipped (specified file is the same
as the current script).
"""
if system_is_windows():
raise NotImplementedError('Self upgrade not supported on Windows.')
import filecmp
import shutil
path = os.path.realpath(path)
self_path = os.path.realpath(__file__)
if filecmp.cmp(path, self_path):
return False
self_stat = os.stat(self_path)
os.chown(path, self_stat.st_uid, self_stat.st_gid)
os.chmod(path, self_stat.st_mode)
shutil.move(path, self_path)
return True
def self_upgrade(include_git=False):
"""Perform in-place self-upgrade.
Parameters
----------
include_git : bool, optional
See `download_latest_googler`. Default is False.
"""
git_ref, path = download_latest_googler(include_git=include_git)
if self_replace(path):
printerr('Upgraded to %s.' % git_ref)
else:
printerr('Already up to date.')
# Miscellaneous functions
def https_proxy_from_environment():
if 'https_proxy' not in os.environ:
return None
proxy = urllib.parse.urlparse(os.environ['https_proxy']).netloc
# urlparse recognizes a netloc only if it is introduced by '//'
# (https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlparse).
# This means a bare host:port like 'localhost:8118' won't be
# recognized. In that case, try again with the '//' prefix.
if not proxy:
proxy = urllib.parse.urlparse('//' + os.environ['https_proxy']).netloc
return proxy if proxy else None
def parse_args(args=None, namespace=None):
"""Parse googler arguments/options.
Parameters
----------
args : list, optional
Arguments to parse. Default is ``sys.argv``.
namespace : argparse.Namespace
Namespace to write to. Default is a new namespace.
Returns
-------
argparse.Namespace
Namespace with parsed arguments / options.
"""
colorstr_env = os.getenv('GOOGLER_COLORS')
argparser = GooglerArgumentParser(description='Google from the command-line.')
addarg = argparser.add_argument
addarg('-s', '--start', type=argparser.nonnegative_int, default=0,
metavar='N', help='start at the Nth result')
addarg('-n', '--count', dest='num', type=argparser.positive_int,
default=10, metavar='N', help='show N results (default 10)')
addarg('-N', '--news', action='store_true',
help='show results from news section')
addarg('-c', '--tld', metavar='TLD',
help="""country-specific search with top-level domain .TLD, e.g., 'in'
for India. Ref: https://en.wikipedia.org/wiki/List_of_Google_domains""")
addarg('-l', '--lang', metavar='LANG', help='display in language LANG')
addarg('-x', '--exact', action='store_true',
help='disable automatic spelling correction')
addarg('-C', '--nocolor', dest='colorize', action='store_false',
help='disable color output')
addarg('--colors', dest='colorstr', type=argparser.is_colorstr,
default=colorstr_env if colorstr_env else 'GKlgxy', metavar='COLORS',
help='set output colors (see man page for details)')
addarg('-j', '--first', '--lucky', dest='lucky', action='store_true',
help='open the first result in web browser and exit')
addarg('-t', '--time', dest='duration', type=argparser.is_duration,
metavar='dN', help='time limit search '
'[h5 (5 hrs), d5 (5 days), w5 (5 weeks), m5 (5 months), y5 (5 years)]')
addarg('-w', '--site', metavar='SITE', help='search a site using Google')
addarg('-p', '--proxy', default=https_proxy_from_environment(),
help='tunnel traffic through an HTTPS proxy (HOST:PORT)')
addarg('--noua', action='store_true', help='disable user agent')
addarg('--notweak', action='store_true',
help='disable TCP optimizations and forced TLS 1.2')
addarg('--json', action='store_true',
help='output in JSON format; implies --exact and --noprompt')
addarg('--show-browser-logs', action='store_true',
help='do not suppress browser output (stdout and stderr)')
addarg('--np', '--noprompt', dest='noninteractive', action='store_true',
help='search and exit, do not prompt')
addarg('keywords', nargs='*', metavar='KEYWORD', help='search keywords')
if ENABLE_SELF_UPGRADE_MECHANISM and not system_is_windows():
addarg('-u', '--upgrade', action='store_true',
help='perform in-place self-upgrade')
addarg('--include-git', action='store_true',
help='when used with --upgrade, upgrade to latest git master')
addarg('-v', '--version', action='version', version=_VERSION_)
addarg('-d', '--debug', action='store_true', help='enable debugging')
return argparser.parse_args(args, namespace)
def main():
global ua
try:
opts = parse_args()
# --json implies --exact
if opts.json:
opts.exact = True
# Set logging level
if opts.debug:
logger.setLevel(logging.DEBUG)
logger.debug('Version %s', _VERSION_)
# Handle self-upgrade
if hasattr(opts, 'upgrade') and opts.upgrade:
self_upgrade(include_git=opts.include_git)
sys.exit(0)
check_stdout_encoding()
if opts.keywords:
try:
# Add cmdline args to readline history
readline.add_history(' '.join(opts.keywords))
except Exception:
pass
# Set colors
if opts.colorize:
colors = Colors(*[COLORMAP[c] for c in opts.colorstr], reset=COLORMAP['x'])
else:
colors = None
Result.colors = colors
GooglerCmd.colors = colors
# Handle browser output suppression
if opts.show_browser_logs:
open_url.suppress_browser_output = False
if opts.noua:
logger.debug('User Agent is disabled')
ua = False
repl = GooglerCmd(opts)
if opts.json or opts.lucky or opts.noninteractive:
# Non-interactive mode
repl.fetch()
if opts.lucky:
if repl.results:
open_url(repl.results[0].url)
else:
print('No results.', file=sys.stderr)
else:
repl.display_results(prelude='', json_output=opts.json)
sys.exit(0)
else:
# Interactive mode
repl.cmdloop()
except Exception as e:
# With debugging on, let the exception through for a traceback;
# otherwise, only print the exception error message.
if logger.isEnabledFor(logging.DEBUG):
raise
else:
logger.error(e)
sys.exit(1)
if __name__ == '__main__':
main()
googler-3.0/googler.1 0000664 0000000 0000000 00000025131 13061270561 0014535 0 ustar 00root root 0000000 0000000 .TH "GOOGLER" "1" "Mar 2017" "Version 3.0" "User Commands"
.SH NAME
googler \- Google from the command-line
.SH SYNOPSIS
.B googler [OPTIONS] [KEYWORD [KEYWORD ...]]
.SH DESCRIPTION
.B googler
is a command-line tool to search Google (Web & News) from the terminal. Google site search works too. \fBgoogler\fR shows the title, URL and text context for each result. Results are fetched in pages. Next or previous page navigation is possible using keyboard shortcuts. Results are indexed and a result URL can be opened in a browser using the index number. There is no configuration file as aliases serve the same purpose for this utility. Supports sequential searches in a single instance.
.PP
.B Features
.PP
* Google Search, Google Site Search, Google News
* Fast and clean (no ads, stray URLs or clutter), custom color
* Navigate result pages from omniprompt, open URLs in browser
* Effortless keyword-based site search with googler @t add-on
* Fetch n results in a go, start at the nth result
* Disable automatic spelling correction and search exact keywords
* Specify duration, country/domain (default: worldwide/.com), language
* Google keywords (e.g. \fIfiletype:mime\fR, \fIsite:somesite.com\fR) support
* Open the first result directly in browser (as in I'm Feeling Lucky)
* Non-stop searches: fire new searches at omniprompt without exiting
* HTTPS proxy, User Agent, TLS 1.2 (default) support
* Man page with examples, completion scripts for Bash, Zsh and Fish
* Minimal dependencies
.SH OPTIONS
.TP
.BI "-h, --help"
Show help text and exit.
.TP
.BI "-s, --start=" N
Start at the \fIN\fRth result.
.TP
.BI "-n, --count=" N
Show \fIN\fR results (default 10).
.TP
.BI "-N, --news"
Show results from news section.
.TP
.BI "-c, --tld=" TLD
Country-specific search with top-level domain \fI.TLD\fR, e.g., \fBin\fR for India (refer \fIhttps://en.wikipedia.org/wiki/List_of_Google_domains\fR for a full list of TLDs).
.TP
.BI "-l, --lang=" LANG
Search for the language \fILANG\fR, e.g., \fBfi\fR for Finnish.
.TP
.B "-x, --exact"
Disable automatic spelling correction. Search exact keywords.
.TP
.B "-C, --nocolor"
Disable color output.
.TP
.BI "--colors=" COLORS
Set output colors. Refer to the \fBCOLORS\fR section below for details.
.TP
.B "-j, --first, --lucky"
Open the first result in a web browser; implies \fB--noprompt\fR. Feeling Lucky?
.TP
.BI "-t, --time=" dN
Time limit search [h5 (5 hrs), d5 (5 days), w5 (5 weeks), m5 (5 months), y5 (5 years)].
.TP
.BI "-w, --site=" SITE
Search a site using Google.
.TP
.BI "-p, --proxy=" PROXY
Tunnel traffic through an HTTPS proxy. \fIPROXY\fR is of the form \fIHOST:PORT\fR. The proxy server must support HTTP CONNECT tunneling and must not block port 443 for the relevant Google hosts. If a proxy is not explicitly given, the \fIhttps_proxy\fR environment variable (if available) is used instead.
.TP
.BI "--noua"
Disable user agent. Results are fetched faster.
.TP
.BI "--notweak"
Disable TCP optimizations. Negotiate Transport Layer Security protocol instead of forcing TLS 1.2 (on Python 3.4 and above). Should be used only in case of connection issues.
.TP
.BI "--json"
Output in JSON format; implies \fB--exact\fR and \fB--noprompt\fR.
.TP
.BI "--show-browser-logs"
Do not suppress browser output when opening result in browser; that is, connect stdout and stderr of the browser to googler's stdout and stderr instead of /dev/null. By default, browser output is suppressed (due to certain graphical browsers spewing messages to console) unless the \fBBROWSER\fR environment variable is a known text-based browser: elinks, links, lynx or w3m.
.TP
.BI "--np, --noprompt"
Perform search and exit; do not prompt for further interactions.
.TP
.BI "-u, --upgrade"
Perform in-place self-upgrade. By default, the latest stable version is used. However, the latest git master is used instead if \fB--include-git\fR is also supplied. This mechanism is not available on Windows (including Cygwin), and if you installed \fBgoogler\fR with a package manager, this mechanism may have been disabled by your packager at packaging or install time.
.TP
.BI "--include-git"
See \fB--upgrade\fR.
.TP
.BI "-v, --version"
Show version number and exit.
.TP
.BI "-d, --debug"
Enable debugging.
.SH OMNIPROMPT KEYS
.TP
.BI "n, p"
Fetch the next or previous set of search results.
.TP
.BI "index"
Open the result corresponding to index in browser.
.TP
.BI "f"
Jump to the first page.
.TP
.BI o " [index ...] [a]"
Open space-separated result indices or all indices, if 'a' is specified, in the browser. Open the current search in the browser, if no arguments.
.TP
.BI g " keywords"
Initiate a new Google search for \fIkeywords\fR with original options. This key should be used to search omniprompt keys (including itself) and indices.
.TP
.BI "q, ^D, double Enter"
Exit googler.
.TP
.BI "?"
Show omniprompt help.
.TP
.BI *
Any other string initiates a new search with original options.
.SH GOOGLER @T
\fBgoogler @t\fR is a convenient add-on to Google Site Search with unique keywords. While \fBgoogler\fR has an integrated option to search a site, it could be simplified further with aliases. The file \fIgoogler_at\fR (https://github.com/jarun/googler/blob/master/auto-completion/googler_at/googler_at) contains a list of website search aliases. To source it, run:
.PP
.IP "" 4
.B source googler_at
.PP
or
.PP
.IP "" 4
.B . googler_at
.PP
With \fBgoogler @t\fR, the following command searches Wikipedia for \fIhexspeak\fR:
.PP
.IP "" 4
.B @w hexspeak
.PP
Other \fBgoogler\fR options can be combined. The shell can be configured to be source the file at start-up for further convenience.
.PP
All the aliases start with the \fB@\fR symbol (hence the name \fBgoogler @t\fR) and there is minimum chance they will conflict with any shell commands. Users can add new aliases to the file.
.SH COLORS
\fBgoogler\fR allows you to customize the color scheme via a six-letter string, reminiscent of BSD \fBLSCOLORS\fR. The six letters represent the colors of
.IP - 2
indices
.PD 0 \" Change paragraph spacing to 0 in the list
.IP - 2
titles
.IP - 2
URLs
.IP - 2
metadata/publishing info (Google News only)
.IP - 2
abstracts
.IP - 2
prompts
.PD 1 \" Restore paragraph spacing
.TP
respectively. The six-letter string is passed in either as the argument to the \fB--colors\fR option, or as the value of the environment variable \fBGOOGLER_COLORS\fR.
.TP
We offer the following colors/styles:
.TS
tab(;) box;
l|l
-|-
l|l.
Letter;Color/Style
a;black
b;red
c;green
d;yellow
e;blue
f;magenta
g;cyan
h;white
i;bright black
j;bright red
k;bright green
l;bright yellow
m;bright blue
n;bright magenta
o;bright cyan
p;bright white
A-H;bold version of the lowercase-letter color
I-P;bold version of the lowercase-letter bright color
x;normal
X;bold
y;reverse video
Y;bold reverse video
.TE
.TP
.TP
The default colors string is \fIGKlgxy\fR, which stands for
.IP - 2
bold bright cyan indices
.PD 0 \" Change paragraph spacing to 0 in the list
.IP - 2
bold bright green titles
.IP - 2
bright yellow URLs
.IP - 2
cyan metadata/publishing info
.IP - 2
normal abstracts
.IP - 2
reverse video prompts
.PD 1 \" Restore paragraph spacing
.TP
Note that
.IP - 2
Bright colors (implemented as \\x1b[90m - \\x1b[97m) may not be available in all color-capable terminal emulators;
.IP - 2
Some terminal emulators draw bold text in bright colors instead;
.IP - 2
Some terminal emulators only distinguish between bold and bright colors via a default-off switch.
.TP
Please consult the manual of your terminal emulator as well as \fIhttps://en.wikipedia.org/wiki/ANSI_escape_code\fR for details.
.SH ENVIRONMENT
.TP
.BI BROWSER
Overrides the default browser. Ref:
.I http://docs.python.org/library/webbrowser.html
.TP
.BI GOOGLER_COLORS
Refer to the \fBCOLORS\fR section.
.TP
.BI https_proxy
Refer to the \fB--proxy\fR option.
.SH EXAMPLES
.PP
.IP 1. 4
Google \fBhello world\fR:
.PP
.EX
.IP
.B googler hello world
.EE
.PP
.IP 2. 4
Fetch \fB15 results\fR updated within last \fB14 months\fR, starting from the \fB3rd result\fR for the string \fBcmdline utility\fR in \fBsite\fR tuxdiary.com:
.PP
.EX
.IP
.B googler -n 15 -s 3 -t m14 -w tuxdiary.com cmdline utility
.EE
.PP
.IP 3. 4
Read recent \fBnews\fR on gadgets:
.PP
.EX
.IP
.B googler -N gadgets
.EE
.PP
.IP 4. 4
Fetch results on IPL cricket from \fBGoogle India\fR server in \fBEnglish\fR:
.PP
.EX
.IP
.B googler -c in -l en IPL cricket
.EE
.PP
.IP 5. 4
Search \fBquoted text\fR:
.PP
.EX
.IP
.B googler it\(rs's a \(rs\(dqbeautiful world\(rs\(dq in spring
.EE
.PP
.IP 6. 4
Search for a \fBspecific file type\fR:
.PP
.EX
.IP
.B googler instrumental filetype:mp3
.EE
.PP
.IP 7. 4
Disable \fBautomatic spelling correction\fR, e.g. fetch results for \fIgoogler\fR instead of \fIgoogle\fR:
.PP
.EX
.IP
.B googler -x googler
.EE
.PP
.IP 8. 4
\fBI'm feeling lucky\fR search:
.PP
.EX
.IP
.B googler -j leather jackets
.EE
.PP
.IP 9. 4
\fBWebsite specific\fR search:
.PP
.EX
.IP
.B googler -w tuxdiary.com hello world
.EE
.PP
.IP "" 4
Site specific search continues at omniprompt. Use the \fBg\fR key to run a regular Google search.
.EE
.PP
.IP 10. 4
Alias to find \fBdefinitions of words\fR:
.PP
.EX
.IP
.B alias define='googler -n 2 define'
.EE
.PP
.IP 11. 4
Look up \fBn\fR, \fBp\fR, \fBo\fR, \fBq\fR, \fBg keywords\fR or a result index at the \fBomniprompt\fR: As the omniprompt recognizes \fBn\fR, \fBp\fR, \fBo\fR, \fBq\fR, \fBg\fR or index strings as commands, you need to prefix them with \fBg\fR, e.g.,
.PP
.EX
.PD 0
.IP
.B g n
.IP
.B g g keywords
.IP
.B g 1
.PD
.EE
.PP
.IP 12. 4
Input and output \fBredirection\fR:
.PP
.EX
.IP
.B googler -C hello world < input > output
.EE
.PP
.IP "" 4
Note that \fI-C\fR is required to avoid printing control characters (for colored output).
.IP 13. 4
\fBPipe\fR output:
.PP
.EX
.IP
.B googler -C hello world | tee output
.EE
.IP 14. 4
Use a \fBcustom color scheme\fR, e.g., one warm color scheme designed for Solarized Dark:
.PP
.EX
.IP
.B googler --colors bjdxxy google
.IP
.B GOOGLER_COLORS=bjdxxy googler google
.EE
.IP 15. 4
Tunnel traffic through an \fBHTTPS proxy\fR, e.g., a local Privoxy instance listening on port 8118:
.PP
.EX
.IP
.B googler --proxy localhost:8118 google
.EE
.PP
.IP "" 4
By default the environment variable \fIhttps_proxy\fR is used, if defined.
.SH AUTHORS
Henri Hakkinen
.br
Arun Prakash Jana
.br
Zhiming Wang
.SH HOME
.I https://github.com/jarun/googler
.SH REPORTING BUGS
.I https://github.com/jarun/googler/issues
.SH LICENSE
Copyright \(co 2008 Henri Hakkinen
.br
Copyright \(co 2015-2017 Arun Prakash Jana
.PP
License GPLv3+: GNU GPL version 3 or later .
.br
This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law.
googler-3.0/tests/ 0000775 0000000 0000000 00000000000 13061270561 0014155 5 ustar 00root root 0000000 0000000 googler-3.0/tests/ci-test-wrapper 0000775 0000000 0000000 00000005643 13061270561 0017141 0 ustar 00root root 0000000 0000000 #!/usr/bin/env bash
set -e
declare here repo_root test_script
here="$(perl -e 'use File::Basename; use Cwd "abs_path"; print dirname(abs_path(@ARGV[0]));' -- "$0")"
repo_root="$here/.."
test_script="$here/test"
export GIT_DIR="$here/../.git"
declare -a watchlist
watchlist=(googler tests)
while [[ $1 == -* ]]; do
case $1 in
-h|--help)
cat <<'EOF'
Usage: ci-test-wrapper [-h|--help] [--monitor PATH [PATH ...]]
googler(1) testing wrapper for CIs.
Options:
-h, --help
Print this help and exit.
--watch PATH [PATH ...]
Additional paths (relative to repository root) to watch. Only run tests
when watched paths have been modified. By default only googler and
tests/ are watched, but sometimes additional paths should be watched
depending on circumstances, e.g., for Travis, .travis.yml should also
be watched. Note that this option consumes all of the remaining command
line arguments.
EOF
exit 1
;;
--watch)
shift
watchlist=( "${watchlist[@]}" "$@" )
shift $#
break
;;
*)
printf '\033[31mError: Unrecognized option %q.\033[0m\n' "$1" >&2
exit 1
;;
esac
shift
done
(( $# > 0 )) && {
printf '\033[31mError: Unrecognized arguments %s.\033[0m\n' "$*" >&2
exit 1
}
# Abort if the CI_SKIP_TEST environment variable is detected.
if [[ -n $CI_SKIP_TEST ]]; then
printf 'Detected $CI_SKIP_TEST. Skipping tests.' >&2
exit
fi
# Diff HEAD against a base commit to see if the changes are worth
# testing. (This check is skipped entirely if the CI_FORCE_TEST environment
# variable is set and non-nil.)
#
# * For a regular branch, diff against HEAD^;
# * For a PR branch, diff against the merge base of HEAD and master.
#
# Currently we use $TRAVIS_PULL_REQUEST to determine whether we're building a
# PR branch. Other criteria may be added if we ever expand to other CIs.
if [[ -z $CI_FORCE_TEST ]]; then
printf 'We are watching the following paths:\n' >&2
printf ' - %s\n' "${watchlist[@]}" >&2
printf '\n' >&2
declare diff_commits diff
if [[ -z ${TRAVIS_PULL_REQUEST+x} || $TRAVIS_PULL_REQUEST == false ]]; then
diff_commits='HEAD^..HEAD'
else
diff_commits='master...HEAD'
fi
diff=$(git -C "$repo_root" diff "$diff_commits" -- "${watchlist[@]}")
if [[ -z $diff ]]; then
printf 'None of the watchlist items changed, skipping tests.\n' >&2
printf 'You may set the $CI_FORCE_TEST environment variable to force testing.\n' >&2
exit 0
else
printf 'Changes to watchlist item(s) detected. Will test.\n\n' >&2
fi
else
printf 'Detected $CI_FORCE_TEST. Skipping necessity checks.\n\n' >&2
fi
# Test googler(1) with $repo_root at the beginning of $PATH (so that googler
# from this repo is picked up).
PATH="$repo_root:$PATH" "$test_script" --ci
googler-3.0/tests/test 0000775 0000000 0000000 00000011252 13061270561 0015063 0 ustar 00root root 0000000 0000000 #!/usr/bin/env bash
set -e
declare quiet exitcode
quiet=0
ci=0
exitcode=0
while [[ $1 == -* ]]; do
case $1 in
--ci)
ci=1
quiet=1
;;
-h|--help)
cat <<'EOF'
Usage: test [options]
Run automated tests of googler(1). googler(1) is expected on $PATH.
Requires shuf(1) from coreutils and /usr/share/dict/words.
Options:
--ci
Same to --quiet, except LF is used instead of CR when printing progress
information. The reason is that CI logs are typically line buffered, so
CR won't flush the output, rendering progress info useless.
-h, --help
Print this help and exit.
-q, --quiet
Suppress googler's output except when a test fails. Some progress info
is still printed to stderr. Note that without this option, this script
is rather verbose.
Environment variables:
NUM_TEST_ITERATIONS
Number of random tests to run. Default is 100.
SLEEP_DURATION
Number of seconds to sleep after each query. Default is 0. You may want
to set this to avoid being blocked by Google for spamming.
EOF
exit 1
;;
-q|--quiet)
quiet=1
;;
*)
printf '\033[31mError: Unrecognized option %q.\033[0m\n' "$1" >&2
exit 1
;;
esac
shift
done
[[ $# -gt 0 ]] && {
printf '\033[31mError: Unrecognized argument %q.\033[0m\n' "$1" >&2
exit 1
}
declare num_rand_words
declare -a predefined_wordlist random_wordlist tld_args lang_args
# A UTF-8 wordlist.
predefined_wordlist=('汉语' 'español' 'português' 'ру́сский язы́к' '日本語' '한국어' 'le français')
# Requires shuf(1).
command -v shuf &>/dev/null || {
printf '\033[31mError: shuf(1) not found.\033[0m\n' >&2
exit 1
}
num_rand_words=10
random_wordlist=( $(shuf -n $num_rand_words /usr/share/dict/words 2>/dev/null) )
[[ ${#random_wordlist[@]} == $num_rand_words ]] || {
printf '\033[31mError: Problem reading random words from /usr/share/dict/words.\033[0m\n' >&2
exit 1
}
# Test googler with the given options, and report error if necessary.
#
# Whether googler's output is suppressed depends on whether the global variable
# quiet is truthy (set by -q, --quiet); when a failure is encountered, the
# global variable exitcode is set to 1, and if quiet was set, the test is rerun
# with output turned on.
test_googler () {
report_error () {
local last_status=$?
declare -g exitcode
local rerun=0
[[ $1 == --rerun ]] && {
rerun=1
shift
}
printf '\033[31mError: googler ' >&2
printf '%q ' "$@" >&2
printf 'failed with status %d.\033[0m\n' $last_status >&2
exitcode=1
(( rerun )) && { googler --noprompt -d "$@"; printf '\n\033[33m[Exit status] %d\033[0m\n' $?; } || :
}
declare -g quiet
if (( quiet )); then
googler --noprompt -d "$@" &>/dev/null || report_error --rerun "$@"
else
printf '\033[34m==> googler ' >&2
printf '%q ' "$@" >&2
printf '\033[0m\n' >&2
googler --noprompt -d "$@" || report_error "$@"
echo
fi
}
# Write a list of configurations to $config_list, and later randomly pick from
# that list. (The reason we don't test them all is that Google would block us
# after thousands of queries.)
declare config_list
config_list="$(mktemp)"
trap 'rm -f "$config_list"' EXIT
for tld in com ar au be br ca ch cz de es 'fi' fr id 'in' it jp kr mx nl ph pl pt ro ru se tw ua uk; do
[[ $tld != com ]] && tld_args=(-c $tld) || tld_args=()
for lang in default de en fr hi ja ko zh; do
[[ $lang != default ]] && lang_args=(-l $lang) || lang_args=()
# Test single word queries.
for keyword in "${predefined_wordlist[@]}" "${random_wordlist[@]}"; do
printf '%s ' "${tld_args[@]}" "${lang_args[@]}" "$keyword"
echo
done
# Test double word queries.
for (( i = 0; i + 1 < num_rand_words; i += 2 )); do
printf '%s ' "${tld_args[@]}" "${lang_args[@]}" \
"${random_wordlist[i]}" "${random_wordlist[i+1]}"
echo
done
done
done >"$config_list"
declare num_rand_configs
num_rand_configs="${NUM_TEST_ITERATIONS:-100}"
counter=0
while read -r args; do
(( counter++ )) || :
printf '\033[32mTest %d/%d\033[0m' $counter $num_rand_configs >&2
(( quiet && !ci )) && printf '\r' >&2 || printf '\n' >&2
test_googler $args # explicit word splitting here, yes
sleep "${SLEEP_DURATION:-0}"
done < <(shuf -n $num_rand_configs "$config_list")
(( exitcode )) || printf '\033[K\033[32mAll passed.\033[0m\n'
exit $exitcode
