Buku
Bookmarks
\ Buku bookmarks
\n
- tag
desc = None
comment_tag = tag.findNextSibling('dd')
if comment_tag:
desc = comment_tag.text[0:comment_tag.text.find('\n')]
self.add_rec(tag['href'], tag.string, ('%s%s%s' %
(DELIM, tag['tags'], DELIM))
if tag.has_attr('tags') else None,
desc, 0, True)
self.conn.commit()
infp.close()
return True
def mergedb(self, path):
'''Merge bookmarks from another Buku database file
:param path: path to DB file to merge
:return: True on success, False on failure
'''
try:
# Connect to input DB
if sys.version_info >= (3, 4, 4):
# Python 3.4.4 and above
indb_conn = sqlite3.connect('file:%s?mode=ro' % path, uri=True)
else:
indb_conn = sqlite3.connect(path)
indb_cur = indb_conn.cursor()
indb_cur.execute('SELECT * FROM bookmarks')
except Exception as e:
logerr(e)
return False
resultset = indb_cur.fetchall()
if resultset:
for row in resultset:
self.add_rec(row[1], row[2], row[3], row[4], row[5], True)
self.conn.commit()
try:
indb_cur.close()
indb_conn.close()
except Exception:
pass
return True
def tnyfy_url(self, index=0, url=None, shorten=True):
'''Shorted a URL using Google URL shortener
:param index: shorten the URL at DB index (int)
:param url: pass a URL (string)
:param shorten: True (default) to shorten, False to expand (boolean)
:return: shortened url string on success, None on failure
'''
if not index and not url:
logerr('Either a valid DB index or URL required')
return None
if index:
self.cur.execute('SELECT url FROM bookmarks WHERE id = ? LIMIT 1',
(index,))
results = self.cur.fetchall()
if not results:
return None
url = results[0][0]
proxies = {
'https': os.environ.get('https_proxy'),
}
urlbase = 'https://tny.im/yourls-api.php?action='
if shorten:
from urllib.parse import quote_plus as qp
_u = '%s%s%s' % (urlbase, 'shorturl&format=simple&url=', qp(url))
else:
_u = '%s%s%s' % (urlbase, 'expand&format=simple&shorturl=', url)
try:
r = requests.post(_u,
headers={
'content-type': 'application/json',
'User-Agent': USER_AGENT
},
proxies=proxies)
except Exception as e:
logerr(e)
return None
if r.status_code != 200:
logerr('[%s] %s', r.status_code, r.reason)
return None
return r.text
def fixtags(self):
'''Undocumented API to fix tags set
in earlier versions. Functionalities:
1. Remove duplicate tags
2. Sort tags
3. Use lower case to store tags
'''
to_commit = False
self.cur.execute('SELECT id, tags FROM bookmarks ORDER BY id ASC')
resultset = self.cur.fetchall()
query = 'UPDATE bookmarks SET tags = ? WHERE id = ?'
for row in resultset:
oldtags = row[1]
if oldtags == ',':
continue
tags = parse_tags([oldtags])
if tags == oldtags:
continue
self.cur.execute(query, (tags, row[0],))
to_commit = True
if to_commit:
self.conn.commit()
def close_quit(self, exitval=0):
'''Close a DB connection and exit
:param exitval: program exit value
'''
if self.conn is not None:
try:
self.cur.close()
self.conn.close()
except Exception:
# ignore errors here, we're closing down
pass
sys.exit(exitval)
class ExtendedArgumentParser(argparse.ArgumentParser):
'''Extend classic argument parser'''
# Print program info
@staticmethod
def program_info(file=sys.stdout):
if sys.platform == 'win32' and file == sys.stdout:
file = sys.stderr
file.write('''
SYMBOLS:
> title
+ comment
# tags
Version %s
Copyright © 2015-2017 %s
License: %s
Webpage: https://github.com/jarun/Buku
''' % (__version__, __author__, __license__))
# Print prompt help
@staticmethod
def prompt_help(file=sys.stdout):
file.write('''
keys:
1-N browse search result indices and/or ranges
a open all results in browser
s keyword [...] search for records with ANY keyword
S keyword [...] search for records with ALL keywords
d match substrings ('pen' matches 'opened')
r expression run a regex search
t [...] search bookmarks by a tag or show tag list
(tag list index fetches bookmarks by tag)
? show this help
q, ^D, double Enter exit buku
''')
# Help
def print_help(self, file=sys.stdout):
super(ExtendedArgumentParser, self).print_help(file)
self.program_info(file)
# ----------------
# Helper functions
# ----------------
def is_bad_url(url):
'''Check if URL is malformed
This API is not bulletproof but works in most cases.
:param url: URL to scan
:return: True or False
'''
# Get the netloc token
netloc = parse_url(url).netloc
if not netloc:
# Try of prepend '//' and get netloc
netloc = parse_url('//' + url).netloc
if not netloc:
return True
logdbg('netloc: %s', netloc)
# netloc cannot start or end with a '.'
if netloc.startswith('.') or netloc.endswith('.'):
return True
# netloc should have at least one '.'
if netloc.rfind('.') < 0:
return True
return False
def is_ignored_mime(url):
'''Check if URL links to ignored mime
Only a 'HEAD' request is made for these URLs
:param url: URL to scan
:return: True or False
'''
for mime in SKIP_MIMES:
if url.lower().endswith(mime):
return True
return False
def get_page_title(resp):
'''Invoke HTML parser and extract title from HTTP response
:param resp: HTTP(S) GET response
:return: title fetched from parsed page
'''
parser = BukuHTMLParser()
try:
parser.feed(resp.data.decode(errors='replace'))
except Exception as e:
# Suppress Exception due to intentional self.reset() in BHTMLParser
if logger.isEnabledFor(logging.DEBUG) \
and str(e) != 'we should not get here!':
logerr('get_page_title(): %s', e)
finally:
return parser.parsed_title
def gen_headers():
'''Generate headers for network connection'''
global myheaders, myproxy
myheaders = {
'Accept-Encoding': 'gzip,deflate',
'User-Agent': USER_AGENT,
'Accept': '*/*',
'Cookie': '',
'DNT': '1'
}
myproxy = os.environ.get('https_proxy')
if myproxy:
try:
url = parse_url(myproxy)
except Exception as e:
logerr(e)
return
# Strip username and password (if present) and update headers
if url.auth:
myproxy = myproxy.replace(url.auth + '@', '')
auth_headers = make_headers(basic_auth=url.auth)
myheaders.update(auth_headers)
logdbg('proxy: [%s]', myproxy)
def get_PoolManager():
'''Creates a pool manager with proxy support, if applicable
:return: ProxyManager if https_proxy is defined, else PoolManager.
'''
if myproxy:
return urllib3.ProxyManager(myproxy, num_pools=1, headers=myheaders)
return urllib3.PoolManager(num_pools=1, headers=myheaders)
def network_handler(url):
'''Handle server connection and redirections
:param url: URL to fetch
:return: (title, recognized mime, bad url) tuple
'''
http_handler = None
page_title = None
resp = None
method = 'GET'
if is_bad_url(url):
return ('', 0, 1)
if is_ignored_mime(url):
method = 'HEAD'
if not myheaders:
gen_headers()
try:
http_handler = get_PoolManager()
while True:
resp = http_handler.request(method, url, timeout=40)
if resp.status == 200:
if method == 'GET':
page_title = get_page_title(resp)
elif resp.status == 403 and url.endswith('/'):
# HTTP response Forbidden
# Handle URLs in the form of https://www.domain.com/
# which fail when trying to fetch resource '/'
# retry without trailing '/'
logdbg('Received status 403: retrying...')
# Remove trailing /
url = url[:-1]
resp.release_conn()
continue
else:
logerr('[%s] %s', resp.status, resp.reason)
if resp:
resp.release_conn()
break
except Exception as e:
logerr('network_handler(): %s', e)
finally:
if http_handler:
http_handler.clear()
if method == 'HEAD':
return ('', 1, 0)
if page_title is None:
return ('', 0, 0)
return (page_title.strip().replace('\n', ''), 0, 0)
def parse_tags(keywords=[]):
'''Format and get tag string from tokens
:param keywords: list of tags
:return: comma-delimited string of tags
:return: just delimiter, if no keywords
:return: None, if keyword is None
'''
if keywords is None:
return None
if not keywords:
return DELIM
tags = DELIM
orig_tags = []
unique_tags = []
# Cleanse and get the tags
tagstr = ' '.join(keywords)
marker = tagstr.find(DELIM)
while marker >= 0:
token = tagstr[0:marker]
tagstr = tagstr[marker + 1:]
marker = tagstr.find(DELIM)
token = token.strip()
if token == '':
continue
tags = '%s%s%s' % (tags, token, DELIM)
tagstr = tagstr.strip()
if tagstr != '':
tags = '%s%s%s' % (tags, tagstr, DELIM)
logdbg('keywords: %s', keywords)
logdbg('parsed tags: [%s]', tags)
if tags == DELIM:
return tags
orig_tags += tags.strip(DELIM).split(DELIM)
for tag in orig_tags:
if tag.lower() not in unique_tags:
# Add unique tags in lower case
unique_tags += (tag.lower(), )
# Sort the tags
sorted_tags = sorted(unique_tags)
# Wrap with delimiter
return '%s%s%s' % (DELIM, DELIM.join(sorted_tags), DELIM)
def taglist_subprompt(obj, msg, noninteractive=False):
'''Additional prompt to show unique tag list
:param obj: a valid instance of BukuDb class
:param msg: sub-prompt message
:param noninteractive: do not seek user input
:return: new command string
'''
unique_tags, dic = obj.get_tag_all()
new_results = True
while True:
if new_results:
if not unique_tags:
count = 0
print('0 tags')
else:
count = 1
for tag in unique_tags:
print('%6d. %s (%d)' % (count, tag, dic[tag]))
count += 1
print()
if noninteractive:
return
try:
nav = read_in(msg)
if not nav:
nav = read_in(msg)
if not nav:
# Quit on double enter
return 'q'
nav = nav.strip()
except EOFError:
return 'q'
if is_int(nav) and int(nav) > 0 and int(nav) < count:
return 't ' + unique_tags[int(nav) - 1]
elif is_int(nav):
print('No matching index %s' % nav)
new_results = False
elif is_int(nav[0]):
print('Invalid input')
new_results = False
elif nav == 't':
new_results = True
continue
elif (nav == 'q' or nav == 'd' or nav == '?' or
nav.startswith('s ') or nav.startswith('S ') or
nav.startswith('r ') or nav.startswith('t ')):
return nav
else:
print('Invalid input')
new_results = False
def prompt(obj, results, noninteractive=False, deep=False, subprompt=False):
'''Show each matching result from a search and prompt
:param obj: a valid instance of BukuDb class
:param results: result set from a DB query
:param noninteractive: do not seek user input
:param deep: use deep search
:param subprompt: jump directly to sub prompt
'''
if not type(obj) is BukuDb:
logerr('Not a BukuDb instance')
return
new_results = True
if colorize:
msg = '\x1b[7mbuku (? for help)\x1b[0m '
else:
msg = 'buku (? for help): '
while True:
if not subprompt:
if new_results:
if results:
count = 0
for row in results:
count += 1
print_record(row, count)
else:
print('0 results')
if noninteractive:
return
try:
nav = read_in(msg)
if not nav:
nav = read_in(msg)
if not nav:
# Quit on double enter
break
nav = nav.strip()
except EOFError:
return
else:
nav = 't'
subprompt = False
# list tags with 't'
if nav == 't':
nav = taglist_subprompt(obj, msg, noninteractive)
if noninteractive:
return
# search ANY match with new keywords
if nav.startswith('s '):
results = obj.searchdb(nav[2:].split(), False, deep)
new_results = True
continue
# search ALL match with new keywords
if nav.startswith('S '):
results = obj.searchdb(nav[2:].split(), True, deep)
new_results = True
continue
# regular expressions search with new keywords
if nav.startswith('r '):
results = obj.searchdb(nav[2:].split(), True, regex=True)
new_results = True
continue
# tag search with new keywords
if nav.startswith('t '):
results = obj.search_by_tag(nav[2:])
new_results = True
continue
# quit with 'q'
if nav == 'q':
return
# toggle deep search with 'd'
if nav == 'd':
deep = not deep
if deep:
print('deep search on')
else:
print('deep search off')
new_results = False
continue
# Show help with '?'
if nav == '?':
ExtendedArgumentParser.prompt_help(sys.stdout)
new_results = False
continue
new_results = False
# Nothing to browse if there are no results
if not results:
print('Not in a search context')
continue
# open all results and re-prompt with 'a'
if nav == 'a':
for index in range(0, count):
browse(results[index][1])
continue
# iterate over white-space separated indices
for nav in nav.split():
if is_int(nav):
index = int(nav) - 1
if index < 0 or index >= count:
print('No matching index %s' % nav)
continue
browse(results[index][1])
elif '-' in nav and is_int(nav.split('-')[0]) \
and is_int(nav.split('-')[1]):
lower = int(nav.split('-')[0])
upper = int(nav.split('-')[1])
if lower > upper:
lower, upper = upper, lower
for index in range(lower-1, upper):
if 0 <= index < count:
browse(results[index][1])
else:
print('No matching index %d' % (index + 1))
else:
print('Invalid input')
break
def print_record(row, idx=0):
'''Print a single DB record
Handles both search result and individual record
:param idx: search result index. If 0, print with DB index
'''
# Start with index and URL
if idx != 0:
pr = ID_str % (idx, row[1], row[0])
else:
pr = ID_DB_str % (row[0], row[1])
# Indicate if record is immutable
if row[5] & 1:
pr = MUTE_str % (pr)
else:
pr = '%s\n' % (pr)
# Append title
if row[2] != '':
pr = TITLE_str % (pr, row[2])
# Append description
if row[4] != '':
pr = DESC_str % (pr, row[4])
# Append tags IF not default (delimiter)
if row[3] != DELIM:
pr = TAG_str % (pr, row[3][1:-1])
print(pr)
def format_json(resultset, single_record=False, field_filter=0):
'''Return results in Json format
:param single_record: indicates only one record
:param field_filter: determines fields to show
:return: record(s) in Json format
'''
if single_record:
marks = {}
for row in resultset:
if field_filter == 1:
marks['uri'] = row[1]
elif field_filter == 2:
marks['uri'] = row[1]
marks['tags'] = row[3][1:-1]
elif field_filter == 3:
marks['title'] = row[2]
else:
marks['index'] = row[0]
marks['uri'] = row[1]
marks['title'] = row[2]
marks['description'] = row[4]
marks['tags'] = row[3][1:-1]
else:
marks = []
for row in resultset:
if field_filter == 1:
record = {'uri': row[1]}
elif field_filter == 2:
record = {'uri': row[1], 'tags': row[3][1:-1]}
elif field_filter == 3:
record = {'title': row[2]}
else:
record = {'index': row[0], 'uri': row[1], 'title': row[2],
'description': row[4], 'tags': row[3][1:-1]}
marks.append(record)
return json.dumps(marks, sort_keys=True, indent=4)
def is_int(string):
'''Check if a string is a digit
:param string: input string
:return: True on success, False on exception
'''
try:
int(string)
return True
except Exception:
return False
def browse(url):
'''Duplicate stdin, stdout (to suppress showing errors
on the terminal) and open URL in default browser
:param url: URL to open
'''
if not parse_url(url).scheme:
# Prefix with 'http://' is no scheme
# Otherwise, opening in browser fails anyway
# We expect http to https redirection
# will happen for https-only websites
logerr('scheme missing in URI, trying http')
url = '%s%s' % ('http://', url)
_stderr = os.dup(2)
os.close(2)
_stdout = os.dup(1)
os.close(1)
fd = os.open(os.devnull, os.O_RDWR)
os.dup2(fd, 2)
os.dup2(fd, 1)
try:
webbrowser.open(url)
except Exception as e:
logerr('browse(): %s', e)
finally:
os.close(fd)
os.dup2(_stderr, 2)
os.dup2(_stdout, 1)
def check_upstream_release():
'''Check and report the latest upstream release version'''
proxies = {
'https': os.environ.get('https_proxy'),
}
try:
r = requests.get(
'https://api.github.com/repos/jarun/buku/tags?per_page=1',
proxies=proxies
)
except Exception as e:
logerr(e)
return
if r.status_code != 200:
logerr('[%s] %s', r.status_code, r.reason)
else:
latest = r.json()[0]['name']
if latest == 'v' + __version__:
print('This is the latest release')
else:
print('Latest upstream release is %s' % latest)
def regexp(expr, item):
'''Perform a regular expression search'''
return re.search(expr, item, re.IGNORECASE) is not None
def read_in(msg):
disable_sigint_handler()
message = None
try:
message = input(msg)
except KeyboardInterrupt:
print('Interrupted.')
enable_sigint_handler()
return message
def sigint_handler(signum, frame):
'''Custom SIGINT handler'''
global interrupted
interrupted = True
print('\nInterrupted.', file=sys.stderr)
# Do a hard exit from here
os._exit(1)
DEFAULT_HANDLER = signal.signal(signal.SIGINT, sigint_handler)
def disable_sigint_handler():
signal.signal(signal.SIGINT, DEFAULT_HANDLER)
def enable_sigint_handler():
signal.signal(signal.SIGINT, sigint_handler)
# ---------------------
# Editor mode functions
# ---------------------
def get_system_editor():
'''Returns default system editor is $EDITOR is set'''
return os.environ.get('EDITOR', 'none')
def to_temp_file_content(url, title_in, tags_in, desc):
'''Generate temporary file content string
:param url: URL to open
:param title_in: string title to add manually
:param tags_in: string of comma-separated tags to add manually
:param desc: string description
:return: lines as newline separated string
'''
strings = []
# URL
strings.extend(['# Lines beginning with "#" will be stripped.\n\
# Add URL in next line (single line).', ])
if url is not None:
strings.append(url)
# TITLE
strings.extend(['# Add TITLE in next line (single line). \
Leave blank to web fetch, "-" for no title.'])
if title_in is None:
title_in = ''
elif title_in == '':
title_in = '-'
strings.append(title_in)
# TAGS
strings.extend(['# Add comma-separated TAGS in next line (single line).'])
strings.append(tags_in.strip(DELIM) if not None else '')
# DESC
strings.append('# Add COMMENTS in next line(s).')
if desc is not None and desc != '':
strings.append(desc)
else:
strings.append('\n')
return '\n'.join(strings)
def parse_temp_file_content(content):
'''Parse and return temporary file content
:param content: string of content
:return: tuple
url: URL to open
title: string title to add manually
tags: string of comma-separated tags to add manually
comments: string description
'''
content = content.split('\n')
content = [c for c in content if len(c) == 0 or c[0] != '#']
if len(content) == 0 or content[0].strip() == '':
print('Edit aborted')
return None
url = content[0]
title = None
if len(content) > 1:
title = content[1]
if title == '':
title = None
elif title == '-':
title = ''
tags = ','
if len(content) > 2:
tags = parse_tags([content[2]])
comments = []
if len(content) > 3:
comments = [c for c in content[3:]]
# need to remove all empty line that are at the end
# and not those in the middle of the text
for i in range(len(comments) - 1, -1, -1):
if comments[i].strip() != '':
break
if i == -1:
comments = []
else:
comments = comments[0:i+1]
comments = '\n'.join(comments)
return url, title, tags, comments
def edit_rec(editor, url, title_in, tags_in, desc):
'''Edit a bookmark record
:param editor: editor to open
:param url: URL to open
:param title_in: string title to add manually
:param tags_in: string of comma-separated tags to add manually
:param desc: string description
:return: parsed content
'''
import tempfile
import subprocess
temp_file_content = to_temp_file_content(url, title_in, tags_in, desc)
fd, tmpfile = tempfile.mkstemp(prefix='buku-edit-')
os.close(fd)
try:
with open(tmpfile, 'w+', encoding='utf-8') as fp:
fp.write(temp_file_content)
fp.flush()
logdbg('Edited content written to %s', tmpfile)
cmd = editor.split(' ')
cmd.append(tmpfile)
subprocess.call(cmd)
with open(tmpfile, 'r', encoding='utf-8') as f:
content = f.read()
os.remove(tmpfile)
except FileNotFoundError:
if os.path.exists(tmpfile):
os.remove(tmpfile)
logerr('Cannot open editor')
else:
logerr('Cannot open tempfile')
return None
parsed_content = parse_temp_file_content(content)
return parsed_content
# Handle piped input
def piped_input(argv, pipeargs=None):
if not sys.stdin.isatty():
pipeargs.extend(argv)
for s in sys.stdin.readlines():
pipeargs.extend(s.split())
# main starts here
def main():
global colorize, ID_str, ID_DB_str, MUTE_str, TITLE_str, DESC_str, TAG_str
title_in = None
tags_in = None
desc_in = None
pipeargs = []
try:
piped_input(sys.argv, pipeargs)
except KeyboardInterrupt:
pass
# If piped input, set argument vector
if pipeargs:
sys.argv = pipeargs
# Setup custom argument parser
argparser = ExtendedArgumentParser(
description='''Powerful command-line bookmark manager. Your mini web!
POSITIONAL ARGUMENTS:
KEYWORD search keywords''',
formatter_class=argparse.RawTextHelpFormatter,
usage='''buku [OPTIONS] [KEYWORD [KEYWORD ...]]''',
add_help=False
)
HIDE = argparse.SUPPRESS
argparser.add_argument('keywords', nargs='*', metavar='KEYWORD', help=HIDE)
# ---------------------
# GENERAL OPTIONS GROUP
# ---------------------
general_grp = argparser.add_argument_group(
title='GENERAL OPTIONS',
description=''' -a, --add URL [tag, ...]
bookmark URL with comma-separated tags
-u, --update [...] update fields of an existing bookmark
accepts indices and ranges
refresh the title, if no edit options
if no arguments:
- update results when used with search
- otherwise refresh all titles
-w, --write [editor|index]
open editor to edit a fresh bookmark
to update by index, EDITOR must be set
-d, --delete [...] remove bookmarks from DB
accepts indices or a single range
if no arguments:
- delete results when used with search
- otherwise delete all bookmarks
-h, --help show this information and exit
-v, --version show the program version and exit''')
addarg = general_grp.add_argument
addarg('-a', '--add', nargs='+', help=HIDE)
addarg('-u', '--update', nargs='*', help=HIDE)
addarg('-w', '--write', nargs='?', const=get_system_editor(), help=HIDE)
addarg('-d', '--delete', nargs='*', help=HIDE)
addarg('-h', '--help', action='store_true', help=HIDE)
addarg('-v', '--version', action='version', version=__version__, help=HIDE)
# ------------------
# EDIT OPTIONS GROUP
# ------------------
edit_grp = argparser.add_argument_group(
title='EDIT OPTIONS',
description=''' --url keyword bookmark link
--tag [+|-] [...] comma-separated tags
clear bookmark tagset, if no arguments
'+' appends to, '-' removes from tagset
-t, --title [...] bookmark title; if no arguments:
-a: do not set title, -u: clear title
-c, --comment [...] description of the bookmark
clears description, if no arguments
--immutable N disable title fetch from web on update
N=0: mutable (default), N=1: immutable''')
addarg = edit_grp.add_argument
addarg('--url', nargs=1, help=HIDE)
addarg('--tag', nargs='*', help=HIDE)
addarg('-t', '--title', nargs='*', help=HIDE)
addarg('-c', '--comment', nargs='*', help=HIDE)
addarg('--immutable', type=int, default=-1, choices={0, 1}, help=HIDE)
# --------------------
# SEARCH OPTIONS GROUP
# --------------------
search_grp = argparser.add_argument_group(
title='SEARCH OPTIONS',
description=''' -s, --sany find records with ANY search keyword
this is the default search option
-S, --sall find records with ALL search keywords
special keywords -
"blank": entries with empty title/tag
"immutable": entries with locked title
--deep match substrings ('pen' matches 'opens')
--sreg run a regex search
--stag search bookmarks by a tag
list all tags, if no search keywords''')
addarg = search_grp.add_argument
addarg('-s', '--sany', action='store_true', help=HIDE)
addarg('-S', '--sall', action='store_true', help=HIDE)
addarg('--sreg', action='store_true', help=HIDE)
addarg('--deep', action='store_true', help=HIDE)
addarg('--stag', action='store_true', help=HIDE)
# ------------------------
# ENCRYPTION OPTIONS GROUP
# ------------------------
crypto_grp = argparser.add_argument_group(
title='ENCRYPTION OPTIONS',
description=''' -l, --lock [N] encrypt DB file with N (> 0, default 8)
hash iterations to generate key
-k, --unlock [N] decrypt DB file with N (> 0, default 8)
hash iterations to generate key''')
addarg = crypto_grp.add_argument
addarg('-k', '--unlock', nargs='?', type=int, const=8, help=HIDE)
addarg('-l', '--lock', nargs='?', type=int, const=8, help=HIDE)
# ----------------
# POWER TOYS GROUP
# ----------------
power_grp = argparser.add_argument_group(
title='POWER TOYS',
description=''' -e, --export file export bookmarks in Firefox format html
export markdown, if file ends with '.md'
format: [title](url), 1 entry per line
use --tag to export only specific tags
-i, --import file import Firefox or Chrome bookmarks html
import markdown, if file ends with '.md'
-m, --merge file add bookmarks from another buku DB file
-p, --print [...] show record details by indices, ranges
print all bookmarks, if no arguments
-f, --format N limit fields in -p or Json search output
N=1: URL, N=2: URL and tag, N=3: title
-r, --replace oldtag [newtag ...]
replace oldtag with newtag everywhere
delete oldtag, if newtag not specified
-j, --json Json formatted output for -p and search
--nc disable color output
--np do not show the prompt, run and exit
-o, --open [...] browse bookmarks by indices and ranges
open a random bookmark, if no arguments
--oa browse all search results immediately
--shorten index|URL fetch shortened url from tny.im service
--expand index|URL expand a tny.im shortened url
--tacit reduce verbosity
--threads N max network connections in full refresh
default N=4, min N=1, max N=10
-V check latest upstream version available
-z, --debug show debug information and verbose logs''')
addarg = power_grp.add_argument
addarg('-e', '--export', nargs=1, help=HIDE)
addarg('-i', '--import', nargs=1, dest='importfile', help=HIDE)
addarg('-m', '--merge', nargs=1, help=HIDE)
addarg('-p', '--print', nargs='*', help=HIDE)
addarg('-f', '--format', type=int, default=0, choices={1, 2, 3}, help=HIDE)
addarg('-r', '--replace', nargs='+', help=HIDE)
addarg('-j', '--json', action='store_true', help=HIDE)
addarg('--nc', action='store_true', help=HIDE)
addarg('--np', action='store_true', help=HIDE)
addarg('-o', '--open', nargs='*', help=HIDE)
addarg('--oa', action='store_true', help=HIDE)
addarg('--shorten', nargs=1, help=HIDE)
addarg('--expand', nargs=1, help=HIDE)
addarg('--tacit', action='store_true', help=HIDE)
addarg('--threads', type=int, default=4, choices=range(1, 11), help=HIDE)
addarg('-V', dest='upstream', action='store_true', help=HIDE)
addarg('-z', '--debug', action='store_true', help=HIDE)
# Undocumented API
addarg('--fixtags', action='store_true', help=HIDE)
# Show help and exit if no arguments
if len(sys.argv) == 1:
argparser.print_help(sys.stdout)
sys.exit(1)
# Parse the arguments
args = argparser.parse_args()
# Show help and exit if help requested
if args.help:
argparser.print_help(sys.stdout)
sys.exit(0)
# Set up debugging
if args.debug:
logger.setLevel(logging.DEBUG)
logdbg('Version %s', __version__)
else:
logging.disable(logging.WARNING)
urllib3.disable_warnings()
# Handle color output preference
if args.nc:
colorize = False
ID_str = '%d. %s [%s]\n'
ID_DB_str = '%d. %s'
MUTE_str = '%s (L)\n'
TITLE_str = '%s > %s\n'
DESC_str = '%s + %s\n'
TAG_str = '%s # %s\n'
# Handle encrypt/decrypt options at top priority
if args.lock is not None:
BukuCrypt.encrypt_file(args.lock)
elif args.unlock is not None:
BukuCrypt.decrypt_file(args.unlock)
# Set up title
if args.title is not None:
if args.title:
title_in = ' '.join(args.title)
else:
title_in = ''
# Set up tags
if args.tag is not None:
if args.tag:
tags_in = args.tag
else:
tags_in = [DELIM, ]
# Set up comment
if args.comment is not None:
if args.comment:
desc_in = ' '.join(args.comment)
else:
desc_in = ''
# Initialize the database and get handles, set verbose by default
bdb = BukuDb(args.json, args.format, not args.tacit,
colorize=not args.nc)
# Editor mode
if args.write is not None:
if args.write == 'none':
logerr('EDITOR is not set')
bdb.close_quit(1)
elif args.write == '0':
logerr('Cannot edit index 0')
bdb.close_quit(1)
if is_int(args.write):
editor = get_system_editor()
if editor == 'none':
logerr('EDITOR must be set to use index with -w')
bdb.close_quit()
idx = int(args.write)
rec = bdb.get_rec_by_id(idx)
if not rec:
logerr('No matching index %d', idx)
bdb.close_quit(1)
result = edit_rec(editor, rec[1], rec[2], rec[3], rec[4])
if result is not None:
url, title, tags, desc = result
bdb.update_rec(idx, url, title, tags, desc)
elif args.add is None:
# Edit and add a new bookmark
# Parse tags into a comma-separated string
if tags_in:
if tags_in[0] == '+':
tags = '+%s' % parse_tags(tags_in[1:])
elif tags_in[0] == '-':
tags = '-%s' % parse_tags(tags_in[1:])
else:
tags = parse_tags(tags_in)
else:
tags = DELIM
result = edit_rec(args.write, '', title_in, tags, desc_in)
if result is not None:
url, title_in, tags, desc_in = result
bdb.add_rec(url, title_in, tags, desc_in, args.immutable)
# Add record
if args.add is not None:
if args.url is not None and args.update is None:
logerr('Bookmark a single URL at a time')
bdb.close_quit(1)
# Parse tags into a comma-separated string
tags = DELIM
keywords = args.add
if tags_in is not None:
if tags_in[0] == '+':
if len(tags_in) > 1:
# The case: buku -a url tag1, tag2 --tag + tag3, tag4
tags_in = tags_in[1:]
# In case of add, args.add may have URL followed by tags
# Add delimiter as url+tags may not end with one
keywords = args.add + [DELIM] + tags_in
else:
keywords = args.add + [DELIM] + tags_in
if len(keywords) > 1:
tags = parse_tags(keywords[1:])
url = args.add[0]
if args.write and not is_int(args.write):
result = edit_rec(args.write, url, title_in, tags, desc_in)
if result is not None:
url, title_in, tags, desc_in = result
bdb.add_rec(url, title_in, tags, desc_in, args.immutable)
# Search record
search_results = None
search_opted = True
update_search_results = False
if args.sany:
# Search URLs, titles, tags for any keyword
search_results = bdb.searchdb(args.keywords, False, args.deep)
elif args.sall:
# Search URLs, titles, tags with all keywords
search_results = bdb.searchdb(args.keywords, True, args.deep)
elif args.sreg:
# Run a regular expression search
search_results = bdb.searchdb(args.keywords, regex=True)
elif args.stag:
# Search bookmarks by tag
if args.keywords:
search_results = bdb.search_by_tag(' '.join(args.keywords))
else:
# Use sub prompt to list all tags
prompt(bdb, None, args.np, subprompt=True)
elif args.keywords:
search_results = bdb.searchdb(args.keywords, False, args.deep)
else:
search_opted = False
if search_results:
oneshot = args.np
to_delete = False
# Open all results in browser right away if args.oa
# is specified. The has priority over delete/update.
# URLs are opened first and updated/deleted later.
if args.oa:
for row in search_results:
browse(row[1])
# In case of search and delete/update,
# prompt should be non-interactive
# delete gets priority over update
if args.delete is not None and not args.delete:
oneshot = True
to_delete = True
elif args.update is not None and not args.update:
oneshot = True
update_search_results = True
if not args.json:
prompt(bdb, search_results, oneshot, args.deep)
else:
# Printing in Json format is non-interactive
print(format_json(search_results, field_filter=args.format))
# Delete search results if opted
if to_delete:
bdb.delete_resultset(search_results)
# Update record
if args.update is not None:
if args.url is not None:
url_in = args.url[0]
else:
url_in = ''
# Parse tags into a comma-separated string
if tags_in:
if tags_in[0] == '+':
tags = '+%s' % parse_tags(tags_in[1:])
elif tags_in[0] == '-':
tags = '-%s' % parse_tags(tags_in[1:])
else:
tags = parse_tags(tags_in)
else:
tags = None
# No arguments to --update, update all
if not args.update:
# Update all records only if search was not opted
if not search_opted:
bdb.update_rec(0, url_in, title_in, tags, desc_in,
args.immutable, args.threads)
elif update_search_results and search_results is not None:
if not args.tacit:
print('Updated results:\n')
pos = len(search_results) - 1
while pos >= 0:
idx = search_results[pos][0]
bdb.update_rec(idx, url_in, title_in, tags, desc_in,
args.immutable, args.threads)
# Commit at every 200th removal
if pos % 200 == 0:
bdb.conn.commit()
pos -= 1
else:
for idx in args.update:
if is_int(idx):
bdb.update_rec(int(idx), url_in, title_in, tags,
desc_in, args.immutable, args.threads)
elif '-' in idx and is_int(idx.split('-')[0]) \
and is_int(idx.split('-')[1]):
lower = int(idx.split('-')[0])
upper = int(idx.split('-')[1])
if lower > upper:
lower, upper = upper, lower
# Update only once if range starts from 0 (all)
if lower == 0:
bdb.update_rec(0, url_in, title_in, tags, desc_in,
args.immutable, args.threads)
else:
for _id in range(lower, upper + 1):
bdb.update_rec(_id, url_in, title_in, tags,
desc_in, args.immutable,
args.threads)
if interrupted:
break
if interrupted:
break
# Delete record
if args.delete is not None:
if not args.delete:
# Attempt delete-all only if search was not opted
if not search_opted:
bdb.cleardb()
elif len(args.delete) == 1 and '-' in args.delete[0]:
vals = str(args.delete[0]).split('-')
if len(vals) == 2 and is_int(vals[0]) and is_int(vals[1]):
if int(vals[0]) == int(vals[1]):
bdb.delete_rec(int(vals[0]))
elif int(vals[0]) < int(vals[1]):
bdb.delete_rec(0, int(vals[0]), int(vals[1]), True)
else:
bdb.delete_rec(0, int(vals[1]), int(vals[0]), True)
else:
logerr('Invalid index or range')
bdb.close_quit(1)
else:
ids = []
# Select the unique indices
for idx in args.delete:
if idx not in ids:
ids += (idx,)
try:
# Index delete order - highest to lowest
ids.sort(key=lambda x: int(x), reverse=True)
for idx in ids:
bdb.delete_rec(int(idx))
except ValueError:
logerr('Invalid index or range')
# Print record
if args.print is not None:
if not args.print:
bdb.print_rec(0)
else:
for idx in args.print:
if is_int(idx):
bdb.print_rec(int(idx))
elif '-' in idx and is_int(idx.split('-')[0]) \
and is_int(idx.split('-')[1]):
lower = int(idx.split('-')[0])
upper = int(idx.split('-')[1])
if lower > upper:
lower, upper = upper, lower
for _id in range(lower, upper + 1):
bdb.print_rec(_id)
else:
logerr('Invalid index or range to print')
bdb.close_quit(1)
# Replace a tag in DB
if args.replace is not None:
if len(args.replace) == 1:
bdb.delete_tag_at_index(0, args.replace[0])
else:
bdb.replace_tag(args.replace[0], args.replace[1:])
# Export bookmarks
if args.export is not None:
if args.tag is None:
bdb.exportdb(args.export[0])
elif not args.tag:
logerr('Missing tag')
else:
bdb.exportdb(args.export[0], args.tag)
# Import bookmarks
if args.importfile is not None:
bdb.importdb(args.importfile[0])
# Merge a database file and exit
if args.merge is not None:
bdb.mergedb(args.merge[0])
# Open URL in browser
if args.open is not None:
if not args.open:
bdb.browse_by_index(0)
else:
for idx in args.open:
if is_int(idx):
bdb.browse_by_index(int(idx))
elif '-' in idx and is_int(idx.split('-')[0]) \
and is_int(idx.split('-')[1]):
lower = int(idx.split('-')[0])
upper = int(idx.split('-')[1])
if lower > upper:
lower, upper = upper, lower
for _id in range(lower, upper + 1):
bdb.browse_by_index(_id)
else:
logerr('Invalid index or range to open')
bdb.close_quit(1)
# Shorten URL
if args.shorten:
if is_int(args.shorten[0]):
shorturl = bdb.tnyfy_url(index=int(args.shorten[0]))
else:
shorturl = bdb.tnyfy_url(url=args.shorten[0])
if shorturl:
print(shorturl)
# Expand URL
if args.expand:
if is_int(args.expand[0]):
url = bdb.tnyfy_url(index=int(args.expand[0]), shorten=False)
else:
url = bdb.tnyfy_url(url=args.expand[0], shorten=False)
if url:
print(url)
# Report upstream version
if args.upstream:
check_upstream_release()
# Fix tags
if args.fixtags:
bdb.fixtags()
# Close DB connection and quit
bdb.close_quit(0)
if __name__ == '__main__':
main()
Buku-2.9/ci-test-wrapper 0000775 0000000 0000000 00000005545 13052573703 0015264 0 ustar 00root root 0000000 0000000 #!/usr/bin/env bash
set -e
declare here repo_root
here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
repo_root="$here"
export GIT_DIR="$here/.git"
declare -a watchlist
watchlist=(buku.py tests)
while [[ $1 == -* ]]; do
case $1 in
-h|--help)
cat <<'EOF'
Usage: ci-test-wrapper [-h|--help] [--monitor PATH [PATH ...]]
buku(1) testing wrapper for CIs.
Options:
-h, --help
Print this help and exit.
--watch PATH [PATH ...]
Additional paths (relative to repository root) to watch. Only run tests
when watched paths have been modified. By default only buku and
tests/ are watched, but sometimes additional paths should be watched
depending on circumstances, e.g., for Travis, .travis.yml should also
be watched. Note that this option consumes all of the remaining command
line arguments.
EOF
exit 1
;;
--watch)
shift
watchlist=( "${watchlist[@]}" "$@" )
shift $#
break
;;
*)
printf '\033[31mError: Unrecognized option %q.\033[0m\n' "$1" >&2
exit 1
;;
esac
shift
done
(( $# > 0 )) && {
printf '\033[31mError: Unrecognized arguments %s.\033[0m\n' "$*" >&2
exit 1
}
# Abort if the CI_SKIP_TEST environment variable is detected.
if [[ -n $CI_SKIP_TEST ]]; then
printf 'Detected $CI_SKIP_TEST. Skipping tests.' >&2
exit
fi
# Diff HEAD against a base commit to see if the changes are worth
# testing. (This check is skipped entirely if the CI_FORCE_TEST environment
# variable is set and non-nil.)
#
# * For a regular branch, diff against HEAD^;
# * For a PR branch, diff against the merge base of HEAD and master.
#
# Currently we use $TRAVIS_PULL_REQUEST to determine whether we're building a
# PR branch. Other criteria may be added if we ever expand to other CIs.
if [[ -z $CI_FORCE_TEST ]]; then
printf 'We are watching the following paths:\n' >&2
printf ' - %s\n' "${watchlist[@]}" >&2
printf '\n' >&2
declare diff_commits diff
if [[ -z ${TRAVIS_PULL_REQUEST+x} || $TRAVIS_PULL_REQUEST == false ]]; then
diff_commits='HEAD^..HEAD'
else
diff_commits='master...HEAD'
fi
diff=$(git -C "$repo_root" diff "$diff_commits" -- "${watchlist[@]}")
if [[ -z $diff ]]; then
printf 'None of the watchlist items changed, skipping tests.\n' >&2
printf 'You may set the $CI_FORCE_TEST environment variable to force testing.\n' >&2
exit 0
else
printf 'Changes to watchlist item(s) detected. Will test.\n\n' >&2
fi
else
printf 'Detected $CI_FORCE_TEST. Skipping necessity checks.\n\n' >&2
fi
# Test buku(1) with $repo_root at the beginning of $PATH (so that buku
# from this repo is picked up).
cd "$here/tests"
PATH="$repo_root:$PATH" python -m pytest test_*.py --cov buku -vv
Buku-2.9/requirements.txt 0000664 0000000 0000000 00000000112 13052573703 0015555 0 ustar 00root root 0000000 0000000 urllib3>=1.13.1
beautifulsoup4>=4.4.1
cryptography>=1.2.3
requests>=2.9.1
Buku-2.9/setup.py 0000664 0000000 0000000 00000003320 13052573703 0014007 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
import re
import sys
from setuptools import setup
if sys.version_info < (3, 3):
print('ERROR: Buku requires at least Python 3.3 to run.')
sys.exit(1)
with open('buku.py', encoding='utf-8') as f:
version = re.search('__version__ = \'([^\']+)\'', f.read()).group(1)
with open('README.md', encoding='utf-8') as f:
long_description = f.read()
setup(
name='buku',
version=version,
description='Powerful command-line bookmark manager. Your mini web!',
long_description=long_description,
author='Arun Prakash Jana',
author_email='engineerarun@gmail.com',
url='https://github.com/jarun/Buku',
license='GPLv3',
platforms=['any'],
py_modules=['buku'],
entry_points={
'console_scripts': ['buku=buku:main']
},
extras_require={
'HTTP': ['urllib3'],
'CRYPTO': ['cryptography'],
'HTML': ['beautifulsoup4'],
'REQUESTS': ['requests']
},
test_suite='tests',
tests_require=['pytest-cov', 'pytest-catchlog'],
keywords='cli bookmarks tag utility',
classifiers=[
'Development Status :: 5 - Production/Stable',
'Environment :: Console',
'Intended Audience :: End Users/Desktop',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Natural Language :: English',
'Operating System :: OS Independent',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3 :: Only',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Topic :: Internet :: WWW/HTTP :: Indexing/Search',
'Topic :: Utilities'
]
)
Buku-2.9/tests/ 0000775 0000000 0000000 00000000000 13052573703 0013441 5 ustar 00root root 0000000 0000000 Buku-2.9/tests/__init__.py 0000664 0000000 0000000 00000000000 13052573703 0015540 0 ustar 00root root 0000000 0000000 Buku-2.9/tests/test_bukuDb.py 0000664 0000000 0000000 00000047304 13052573703 0016276 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
#
# Unit test cases for buku
#
import os
import re
import sqlite3
from genericpath import exists
from tempfile import TemporaryDirectory
import pytest
import unittest
from unittest import mock as mock
from buku import BukuDb, parse_tags, prompt
TEST_TEMP_DIR_OBJ = TemporaryDirectory(prefix='bukutest_')
TEST_TEMP_DIR_PATH = TEST_TEMP_DIR_OBJ.name
TEST_TEMP_DBDIR_PATH = os.path.join(TEST_TEMP_DIR_PATH, 'buku')
TEST_TEMP_DBFILE_PATH = os.path.join(TEST_TEMP_DBDIR_PATH, 'bookmarks.db')
TEST_BOOKMARKS = [
['http://slashdot.org',
'SLASHDOT',
parse_tags(['old,news']),
"News for old nerds, stuff that doesn't matter"],
['http://www.zażółćgęśląjaźń.pl/',
'ZAŻÓŁĆ',
parse_tags(['zażółć,gęślą,jaźń']),
"Testing UTF-8, zażółć gęślą jaźń."],
['https://test.com:8080',
'test',
parse_tags(['test,tes,est,es']),
"a case for replace_tag test"],
]
@pytest.fixture()
def setup():
os.environ['XDG_DATA_HOME'] = TEST_TEMP_DIR_PATH
# start every test from a clean state
if exists(TEST_TEMP_DBFILE_PATH):
os.remove(TEST_TEMP_DBFILE_PATH)
class TestBukuDb(unittest.TestCase):
def setUp(self):
os.environ['XDG_DATA_HOME'] = TEST_TEMP_DIR_PATH
# start every test from a clean state
if exists(TEST_TEMP_DBFILE_PATH):
os.remove(TEST_TEMP_DBFILE_PATH)
self.bookmarks = TEST_BOOKMARKS
self.bdb = BukuDb()
def tearDown(self):
os.environ['XDG_DATA_HOME'] = TEST_TEMP_DIR_PATH
# @unittest.skip('skipping')
def test_get_default_dbdir(self):
dbdir_expected = TEST_TEMP_DBDIR_PATH
dbdir_local_expected = os.path.join(os.path.expanduser('~'), '.local', 'share', 'buku')
dbdir_relative_expected = os.path.abspath('.')
# desktop linux
self.assertEqual(dbdir_expected, BukuDb.get_default_dbdir())
# desktop generic
os.environ.pop('XDG_DATA_HOME')
self.assertEqual(dbdir_local_expected, BukuDb.get_default_dbdir())
# no desktop
# -- home is defined differently on various platforms.
# -- keep a copy and set it back once done
originals = {}
for env_var in ['HOME', 'HOMEPATH', 'HOMEDIR']:
try:
originals[env_var] = os.environ.pop(env_var)
except KeyError:
pass
self.assertEqual(dbdir_relative_expected, BukuDb.get_default_dbdir())
for key, value in originals.items():
os.environ[key] = value
# # not sure how to test this in nondestructive manner
# def test_move_legacy_dbfile(self):
# self.fail()
# @unittest.skip('skipping')
def test_initdb(self):
if exists(TEST_TEMP_DBFILE_PATH):
os.remove(TEST_TEMP_DBFILE_PATH)
self.assertIs(False, exists(TEST_TEMP_DBFILE_PATH))
conn, curr = BukuDb.initdb()
self.assertIsInstance(conn, sqlite3.Connection)
self.assertIsInstance(curr, sqlite3.Cursor)
self.assertIs(True, exists(TEST_TEMP_DBFILE_PATH))
curr.close()
conn.close()
# @unittest.skip('skipping')
def test_get_rec_by_id(self):
for bookmark in self.bookmarks:
# adding bookmark from self.bookmarks
self.bdb.add_rec(*bookmark)
# the expected bookmark
expected = (1, 'http://slashdot.org', 'SLASHDOT', ',news,old,',
"News for old nerds, stuff that doesn't matter", 0)
bookmark_from_db = self.bdb.get_rec_by_id(1)
# asserting bookmark matches expected
self.assertEqual(expected, bookmark_from_db)
# asserting None returned if index out of range
self.assertIsNone(self.bdb.get_rec_by_id(len(self.bookmarks[0]) + 1))
# @unittest.skip('skipping')
def test_get_rec_id(self):
for idx, bookmark in enumerate(self.bookmarks):
# adding bookmark from self.bookmarks to database
self.bdb.add_rec(*bookmark)
# asserting index is in order
idx_from_db = self.bdb.get_rec_id(bookmark[0])
self.assertEqual(idx + 1, idx_from_db)
# asserting -1 is returned for nonexistent url
idx_from_db = self.bdb.get_rec_id("http://nonexistent.url")
self.assertEqual(-1, idx_from_db)
# @unittest.skip('skipping')
def test_add_rec(self):
for bookmark in self.bookmarks:
# adding bookmark from self.bookmarks to database
self.bdb.add_rec(*bookmark)
# retrieving bookmark from database
index = self.bdb.get_rec_id(bookmark[0])
from_db = self.bdb.get_rec_by_id(index)
self.assertIsNotNone(from_db)
# comparing data
for pair in zip(from_db[1:], bookmark):
self.assertEqual(*pair)
# TODO: tags should be passed to the api as a sequence...
# @unittest.skip('skipping')
def test_update_rec(self):
old_values = self.bookmarks[0]
new_values = self.bookmarks[1]
# adding bookmark and getting index
self.bdb.add_rec(*old_values)
index = self.bdb.get_rec_id(old_values[0])
# updating with new values
self.bdb.update_rec(index, *new_values)
# retrieving bookmark from database
from_db = self.bdb.get_rec_by_id(index)
self.assertIsNotNone(from_db)
# checking if values are updated
for pair in zip(from_db[1:], new_values):
self.assertEqual(*pair)
# @unittest.skip('skipping')
def test_append_tag_at_index(self):
for bookmark in self.bookmarks:
self.bdb.add_rec(*bookmark)
# tags to add
old_tags = self.bdb.get_rec_by_id(1)[3]
new_tags = ",foo,bar,baz"
self.bdb.append_tag_at_index(1, new_tags)
# updated list of tags
from_db = self.bdb.get_rec_by_id(1)[3]
# checking if new tags were added to the bookmark
self.assertTrue(split_and_test_membership(new_tags, from_db))
# checking if old tags still exist
self.assertTrue(split_and_test_membership(old_tags, from_db))
# @unittest.skip('skipping')
def test_append_tag_at_all_indices(self):
for bookmark in self.bookmarks:
self.bdb.add_rec(*bookmark)
# tags to add
new_tags = ",foo,bar,baz"
# record of original tags for each bookmark
old_tagsets = {i: self.bdb.get_rec_by_id(i)[3] for i in inclusive_range(1, len(self.bookmarks))}
with mock.patch('builtins.input', return_value='y'):
self.bdb.append_tag_at_index(0, new_tags)
# updated tags for each bookmark
from_db = [(i, self.bdb.get_rec_by_id(i)[3]) for i in inclusive_range(1, len(self.bookmarks))]
for index, tagset in from_db:
# checking if new tags added to bookmark
self.assertTrue(split_and_test_membership(new_tags, tagset))
# checking if old tags still exist for boomark
self.assertTrue(split_and_test_membership(old_tagsets[index], tagset))
# @unittest.skip('skipping')
def test_delete_tag_at_index(self):
# adding bookmarks
for bookmark in self.bookmarks:
self.bdb.add_rec(*bookmark)
get_tags_at_idx = lambda i: self.bdb.get_rec_by_id(i)[3]
# list of two-tuples, each containg bookmark index and corresponding tags
tags_by_index = [(i, get_tags_at_idx(i)) for i in inclusive_range(1, len(self.bookmarks))]
for i, tags in tags_by_index:
# get the first tag from the bookmark
to_delete = re.match(',.*?,', tags).group(0)
self.bdb.delete_tag_at_index(i, to_delete)
# get updated tags from db
from_db = get_tags_at_idx(i)
self.assertNotIn(to_delete, from_db)
# @unittest.skip('skipping')
def test_refreshdb(self):
self.bdb.add_rec("https://www.google.com/ncr", "?")
self.bdb.refreshdb(1, 1)
from_db = self.bdb.get_rec_by_id(1)
self.assertEqual(from_db[2], "Google")
# @unittest.skip('skipping')
def test_searchdb(self):
# adding bookmarks
for bookmark in self.bookmarks:
self.bdb.add_rec(*bookmark)
get_first_tag = lambda x: ''.join(x[2].split(',')[:2])
for i, bookmark in enumerate(self.bookmarks):
tag_search = get_first_tag(bookmark)
# search by the domain name for url
url_search = re.match('https?://(.*)?\..*', bookmark[0]).group(1)
title_search = bookmark[1]
# Expect a five-tuple containing all bookmark data
# db index, URL, title, tags, description
expected = [(i + 1,) + tuple(bookmark)]
# search db by tag, url (domain name), and title
for keyword in (tag_search, url_search, title_search):
with mock.patch('buku.prompt'):
# search by keyword
results = self.bdb.searchdb([keyword])
self.assertEqual(results, expected)
# @unittest.skip('skipping')
def test_search_by_tag(self):
# adding bookmarks
for bookmark in self.bookmarks:
self.bdb.add_rec(*bookmark)
with mock.patch('buku.prompt'):
get_first_tag = lambda x: ''.join(x[2].split(',')[:2])
for i in range(len(self.bookmarks)):
# search for bookmark with a tag that is known to exist
results = self.bdb.search_by_tag(get_first_tag(self.bookmarks[i]))
# Expect a five-tuple containing all bookmark data
# db index, URL, title, tags, description
expected = [(i + 1,) + tuple(self.bookmarks[i])]
self.assertEqual(results, expected)
# @unittest.skip('skipping')
def test_search_and_open_in_broswer_by_range(self):
# adding bookmarks
for bookmark in self.bookmarks:
self.bdb.add_rec(*bookmark)
# simulate user input, select range of indices 1-3
index_range = '1-%s' % len(self.bookmarks)
with mock.patch('builtins.input', side_effect=[index_range]):
with mock.patch('buku.browse') as mock_browse:
try:
# search the db with keywords from each bookmark
# searching using the first tag from bookmarks
get_first_tag = lambda x: x[2].split(',')[1]
results = self.bdb.searchdb([get_first_tag(bm) for bm in self.bookmarks])
prompt(self.bdb, results)
except StopIteration:
# catch exception thrown by reaching the end of the side effect iterable
pass
# collect arguments passed to browse
arg_list = [args[0] for args, _ in mock_browse.call_args_list]
# expect a list of one-tuples that are bookmark URLs
expected = [x[0] for x in self.bookmarks]
# checking if browse called with expected arguments
self.assertEqual(arg_list, expected)
# @unittest.skip('skipping')
def test_search_and_open_all_in_browser(self):
# adding bookmarks
for bookmark in self.bookmarks:
self.bdb.add_rec(*bookmark)
# simulate user input, select 'a' to open all bookmarks in results
with mock.patch('builtins.input', side_effect=['a']):
with mock.patch('buku.browse') as mock_browse:
try:
# search the db with keywords from each bookmark
# searching using the first tag from bookmarks
get_first_tag = lambda x: x[2].split(',')[1]
results = self.bdb.searchdb([get_first_tag(bm) for bm in self.bookmarks[:2]])
prompt(self.bdb, results)
except StopIteration:
# catch exception thrown by reaching the end of the side effect iterable
pass
# collect arguments passed to browse
arg_list = [args[0] for args, _ in mock_browse.call_args_list]
# expect a list of one-tuples that are bookmark URLs
expected = [x[0] for x in self.bookmarks][:2]
# checking if browse called with expected arguments
self.assertEqual(arg_list, expected)
# @unittest.skip('skipping')
def test_delete_rec(self):
# adding bookmark and getting index
self.bdb.add_rec(*self.bookmarks[0])
index = self.bdb.get_rec_id(self.bookmarks[0][0])
# deleting bookmark
self.bdb.delete_rec(index)
# asserting it doesn't exist
from_db = self.bdb.get_rec_by_id(index)
self.assertIsNone(from_db)
# @unittest.skip('skipping')
def test_delete_rec_yes(self):
# checking that "y" response causes delete_rec to return True
with mock.patch('builtins.input', return_value='y'):
self.assertTrue(self.bdb.delete_rec(0))
# @unittest.skip('skipping')
def test_delete_rec_no(self):
# checking that non-"y" response causes delete_rec to return None
with mock.patch('builtins.input', return_value='n'):
self.assertFalse(self.bdb.delete_rec(0))
# @unittest.skip('skipping')
def test_cleardb(self):
# adding bookmarks
self.bdb.add_rec(*self.bookmarks[0])
# deleting all bookmarks
with mock.patch('builtins.input', return_value='y'):
self.bdb.cleardb()
# assert table has been dropped
with self.assertRaises(sqlite3.OperationalError) as ctx_man:
self.bdb.get_rec_by_id(0)
err_msg = str(ctx_man.exception)
self.assertEqual(err_msg, 'no such table: bookmarks')
# @unittest.skip('skipping')
def test_replace_tag(self):
indices = []
for bookmark in self.bookmarks:
# adding bookmark, getting index
self.bdb.add_rec(*bookmark)
index = self.bdb.get_rec_id(bookmark[0])
indices += [index]
# replacing tags
with mock.patch('builtins.input', return_value='y'):
self.bdb.replace_tag("news", ["__01"])
with mock.patch('builtins.input', return_value='y'):
self.bdb.replace_tag("zażółć", ["__02,__03"])
# replacing tag which is also a substring of other tag
with mock.patch('builtins.input', return_value='y'):
self.bdb.replace_tag("es", ["__04"])
# removing tags
with mock.patch('builtins.input', return_value='y'):
self.bdb.replace_tag("gęślą")
with mock.patch('builtins.input', return_value='y'):
self.bdb.replace_tag("old")
# removing non-existent tag
with mock.patch('builtins.input', return_value='y'):
self.bdb.replace_tag("_")
# removing nonexistent tag which is also a substring of other tag
with mock.patch('builtins.input', return_value='y'):
self.bdb.replace_tag("e")
for url, title, _, _ in self.bookmarks:
# retrieving from db
index = self.bdb.get_rec_id(url)
from_db = self.bdb.get_rec_by_id(index)
# asserting tags were replaced
if title == "SLASHDOT":
self.assertEqual(from_db[3], parse_tags(["__01"]))
elif title == "ZAŻÓŁĆ":
self.assertEqual(from_db[3], parse_tags(["__02,__03,jaźń"]))
elif title == "test":
self.assertEqual(from_db[3], parse_tags(["test,tes,est,__04"]))
# def test_browse_by_index(self):
# self.fail()
# @unittest.skip('skipping')
def test_close_quit(self):
# quitting with no args
try:
self.bdb.close_quit()
except SystemExit as err:
self.assertEqual(err.args[0], 0)
# quitting with custom arg
try:
self.bdb.close_quit(1)
except SystemExit as err:
self.assertEqual(err.args[0], 1)
# def test_import_bookmark(self):
# self.fail()
def test_print_rec(capsys, caplog, setup):
bdb = BukuDb()
out, err = capsys.readouterr()
# calling with nonexistent index
bdb.print_rec(1)
out, err = capsys.readouterr()
for record in caplog.records():
assert record.levelname == "ERROR"
assert record.getMessage() == "No matching index 1"
assert (out, err) == ('', '')
# adding bookmarks
bdb.add_rec("http://full-bookmark.com", "full", parse_tags(['full,bookmark']), "full bookmark")
bdb.add_rec("http://blank-title.com", "", parse_tags(['blank,title']), "blank title")
bdb.add_rec("http://empty-tags.com", "empty tags", parse_tags(['']), "empty tags")
bdb.add_rec("http://all-empty.com", "", parse_tags(['']), "all empty")
out, err = capsys.readouterr()
# printing first bookmark
bdb.print_rec(1)
out, err = capsys.readouterr()
assert out == "\x1b[1m\x1b[93m1. \x1b[0m\x1b[92mhttp://full-bookmark.com\x1b[0m\n \x1b[91m>\x1b[0m full\n \x1b[91m+\x1b[0m full bookmark\n \x1b[91m#\x1b[0m bookmark,full\n\n"
assert err == ''
# printing all bookmarks
bdb.print_rec(0)
out, err = capsys.readouterr()
assert out == "\x1b[1m\x1b[93m1. \x1b[0m\x1b[92mhttp://full-bookmark.com\x1b[0m\n \x1b[91m>\x1b[0m full\n \x1b[91m+\x1b[0m full bookmark\n \x1b[91m#\x1b[0m bookmark,full\n\n\x1b[1m\x1b[93m2. \x1b[0m\x1b[92mhttp://blank-title.com\x1b[0m\n \x1b[91m+\x1b[0m blank title\n \x1b[91m#\x1b[0m blank,title\n\n\x1b[1m\x1b[93m3. \x1b[0m\x1b[92mhttp://empty-tags.com\x1b[0m\n \x1b[91m>\x1b[0m empty tags\n \x1b[91m+\x1b[0m empty tags\n\n\x1b[1m\x1b[93m4. \x1b[0m\x1b[92mhttp://all-empty.com\x1b[0m\n \x1b[91m+\x1b[0m all empty\n\n"
assert err == ''
# printing all bookmarks with empty fields
results = bdb.searchdb(['blank'], True)
prompt(bdb, results, True)
out, err = capsys.readouterr()
assert out == "\x1b[1m\x1b[93m1. \x1b[0m\x1b[92mhttp://blank-title.com\x1b[0m \x1b[1m[2]\x1b[0m\n \x1b[91m+\x1b[0m blank title\n \x1b[91m#\x1b[0m blank,title\n\n\x1b[1m\x1b[93m2. \x1b[0m\x1b[92mhttp://empty-tags.com\x1b[0m \x1b[1m[3]\x1b[0m\n \x1b[91m>\x1b[0m empty tags\n \x1b[91m+\x1b[0m empty tags\n\n\x1b[1m\x1b[93m3. \x1b[0m\x1b[92mhttp://all-empty.com\x1b[0m \x1b[1m[4]\x1b[0m\n \x1b[91m+\x1b[0m all empty\n\n"
assert err == ''
def test_list_tags(capsys, setup):
bdb = BukuDb()
# adding bookmarks
bdb.add_rec("http://one.com", "", parse_tags(['cat,ant,bee,1']), "")
bdb.add_rec("http://two.com", "", parse_tags(['Cat,Ant,bee,1']), "")
bdb.add_rec("http://three.com", "", parse_tags(['Cat,Ant,3,Bee,2']), "")
# listing tags, asserting output
out, err = capsys.readouterr()
prompt(bdb, None, True, subprompt=True)
out, err = capsys.readouterr()
assert out == " 1. 1 (2)\n 2. 2 (1)\n 3. 3 (1)\n 4. ant (3)\n 5. bee (3)\n 6. cat (3)\n\n"
assert err == ''
def test_compactdb(setup):
bdb = BukuDb()
# adding bookmarks
for bookmark in TEST_BOOKMARKS:
bdb.add_rec(*bookmark)
# manually deleting 2nd index from db, calling compactdb
bdb.cur.execute('DELETE FROM bookmarks WHERE id = ?', (2,))
bdb.compactdb(2)
# asserting bookmarks have correct indices
assert bdb.get_rec_by_id(1) == (1, 'http://slashdot.org', 'SLASHDOT', ',news,old,', "News for old nerds, stuff that doesn't matter", 0)
assert bdb.get_rec_by_id(2) == (2, 'https://test.com:8080', 'test', ',es,est,tes,test,', 'a case for replace_tag test', 0)
assert bdb.get_rec_by_id(3) is None
# Helper functions for testcases
def split_and_test_membership(a, b):
# :param a, b: comma separated strings to split
# test everything in a in b
return all(x in b.split(',') for x in a.split(','))
def inclusive_range(start, end):
return range(start, end + 1)
if __name__ == "__main__":
unittest.main()
Buku-2.9/tests/test_helpers.py 0000664 0000000 0000000 00000003660 13052573703 0016521 0 ustar 00root root 0000000 0000000 #!/usr/bin/env python3
import os
import signal
import unittest
from buku import is_int, parse_tags
class TestHelpers(unittest.TestCase):
# @unittest.skip('skipping')
def test_parse_tags(self):
# call with None
parsed = parse_tags(None)
self.assertIsNone(parsed)
# call with empty list
parsed = parse_tags([])
self.assertEqual(parsed, ",")
# empty tags
parsed = parse_tags([",,,,,"])
self.assertEqual(parsed, ",")
# sorting tags
parsed = parse_tags(["z_tag,a_tag,n_tag"])
self.assertEqual(parsed, ",a_tag,n_tag,z_tag,")
# whitespaces
parsed = parse_tags([" a tag , , , ,\t,\n,\r,\x0b,\x0c"])
self.assertEqual(parsed, ",a tag,")
# duplicates, excessive spaces
parsed = parse_tags(["tag,tag, tag, tag,tag , tag "])
self.assertEqual(parsed, ",tag,")
# escaping quotes
parsed = parse_tags(["\"tag\",\'tag\',tag"])
self.assertEqual(parsed, ",\"tag\",\'tag\',tag,")
# combo
parsed = parse_tags([",,z_tag, a tag ,\t,,, ,n_tag ,n_tag, a_tag, \na tag ,\r, \"a_tag\""])
self.assertEqual(parsed, ",\"a_tag\",a tag,a_tag,n_tag,z_tag,")
# @unittest.skip('skipping')
def test_is_int(self):
self.assertTrue(is_int('0'))
self.assertTrue(is_int('1'))
self.assertTrue(is_int('-1'))
self.assertFalse(is_int(''))
self.assertFalse(is_int('one'))
# This test fails because we use os._exit() now
@unittest.skip('skipping')
def test_sigint_handler(capsys):
try:
# sending SIGINT to self
os.kill(os.getpid(), signal.SIGINT)
except SystemExit as error:
out, err = capsys.readouterr()
# assert exited with 1
assert error.args[0] == 1
# assert proper error message
assert out == ''
assert err == "\nInterrupted.\n"
if __name__ == "__main__":
unittest.main()
Buku-2.9/tools/ 0000775 0000000 0000000 00000000000 13052573703 0013437 5 ustar 00root root 0000000 0000000 Buku-2.9/tools/makedeb 0000775 0000000 0000000 00000020136 13052573703 0014757 0 ustar 00root root 0000000 0000000 #!/usr/bin/env zsh
# Automatically make .deb package from a commit or a tag.
#
# Prerequisites:
# zsh, build-essential, devscripts, debhelper (>= 9)
#
# Reference: https://wiki.debian.org/IntroDebianPackaging.
setopt errexit noshwordsplit nobashrematch
[[ -n $DEBUG ]] && setopt xtrace
################### SET UP ENVIRONMENT AND BASE DIRECTORIES ####################
[[ -z $DEBFULLNAME ]] && DEBFULLNAME='Arun Prakash Jana'
[[ -z $DEBEMAIL ]] && DEBEMAIL='engineerarun@gmail.com'
[[ -z $TZ ]] && TZ='Asia/Kolkata'
export DEBFULLNAME DEBEMAIL TZ
here=$0:A:h
repodir=$here/..
builddir=$here/../build
distdir=$here/../dist
export GIT_DIR=$repodir/.git
################################# SET UP TRAPS #################################
# Trap SIGUSR1: Abort program when functions called from within command
# substitutions in heredocs fail.
trap 'print_error "Encountered problem inside cmdsubst at line $LINENO."; exit 1' SIGUSR1
export NOTIFY_PID=$$
############################### HELPER FUNCTIONS ###############################
print_error () print -R $'\e[31m'"Error: $*"$'\e[0m' >&2
print_warning () print -R $'\e[33m'"Warning: $*"$'\e[0m' >&2
# Usage: apt_package_version
apt_package_version () { local version="${$(apt-cache show $1 | grep '^Version')#Version: }" || { print_error "Version info not available for package ${(q-)1}." [[ -n $NOTIFY_PID ]] && kill -SIGUSR1 $NOTIFY_PID exit 1 } printf %s $version } debian_policy_version () { local full_version="$(apt_package_version debian-policy)" local match [[ $full_version =~ ^(([0-9]+\.){2}[0-9]+) ]] || { print_error "Invalid debian-policy version ${(q-)full_version}." [[ -n $NOTIFY_PID ]] && kill -SIGUSR1 $NOTIFY_PID exit 1 } printf %s $match[1] } # Git helpers # Usage: git_normalize_commitish # # Normalize a commit-ish to a tag name if the commit-ish refers to a tag or # tagged commit; otherwise to a SHA (via git-rev-parse). git_normalize_commitish () { local tagname commitsha if tagname=$(git describe --exact --tags $1 2>/dev/null); then printf %s $tagname else commitsha=$(git rev-parse --verify --quite $1) || { print_error "Unable to parse ${(q-)1} as a git commit." [[ -n $NOTIFY_PID ]] && kill -SIGUSR1 $NOTIFY_PID exit 1 } printf %s $commitsha fi } # Usage: git_ref_is_tag # # Returns 0 (is tag) or 1 (not tag). git_ref_is_tag () { git show-ref --quiet --verify refs/tags/$1 } # Usage: git_commitish_timestamp # # Expects a normalized commit-ish (see git_normalize_commitish) as input, and # outputs a timestamp. For a tag, the timestamp is the tagger date. For a # commit, the timestamp is the committer date. git_commitish_timestamp () { if git_ref_is_tag $1; then local date="$(git for-each-ref --format='%(taggerdate)' refs/tags/$1)" # The date returned by git-for-each-ref looks like `Sat Apr 23 10:38:27 # 2016 +0530', which isn't recognized by date(1) from coreutils. Need a # hack to turn `+0530' into `U+0530' (same for -). # # Note that date will be empty if the tag at question is a lightweight # tag (i.e., non-annotated) because they don't carry any tagger # information; in that case we simply fall back to the committer. if [[ -n $date ]]; then date -d ${${date// -/ U-}// +/ U+} +%s return fi fi git rev-list --format=format:%ct --max-count=1 $1 | tail -n1 } ######################### PARSE COMMAND LINE ARGUMENTS ######################### tag_only=0 while [[ $1 == -* ]]; do case $1 in -h|--help) cat >&2 < ] Make a deb package from a git commit-ish, which defaults to HEAD. Options: -h, --help Print this help and exit. --tag-only Only make deb if HEAD (or , if specified) is a tag or a tagged commit. EOF exit 1 ;; --tag-only) tag_only=1 ;; --) shift break ;; *) print_error "Unknown option ${(q-)1}." exit 1 ;; esac shift done [[ -n $1 ]] && { commitish=$1; shift } || commitish=HEAD (( $# > 0 )) && { print_error 'Unrecognized arguments' ${(q-)@} exit 1 } ##################################### MAIN ##################################### mkdir -p $builddir $distdir cd $builddir normalized_commitish=$(git_normalize_commitish $commitish) git_ref_is_tag $normalized_commitish && commitish_is_tag=1 || commitish_is_tag=0 (( tag_only && !commitish_is_tag )) && { print_warning "${(q-)commitish} is not a tag or tagged commit, skipping build." exit } pkgname=buku version="${$(git describe --tags $commitish)#v}" # Quoting just to make sh-mode happy [[ -n $version ]] || { print_error 'Failed to extract version information.' exit 1 } debrevision=1 creation_timestamp=$(git_commitish_timestamp $normalized_commitish) upstream_tarball=$builddir/${pkgname}_${version}.orig.tar.gz buildsubdirname=${pkgname}-${version} git -C $repodir archive --format=tar.gz --prefix=$buildsubdirname/ --output=$upstream_tarball $commitish . rm -rf $buildsubdirname tar xf $upstream_tarball cd $buildsubdirname mkdir debian # Write debian/changelog if (( commitish_is_tag )); then # Tag -- point to the release changelog_url=https://github.com/jarun/Buku/releases/tag/$normalized_commitish else # Just a commit -- point to the list of commits in the tree changelog_url=https://github.com/jarun/Buku/commits/$normalized_commitish fi cat >debian/changelog < $(date --rfc-2822 --date=@$creation_timestamp) EOF # Alternatively, use dch to create changelog interactively: # dch --create -v ${version}-${debrevision} --package $pkgname # Write debian/compat cat >debian/compat <<'EOF' 9 EOF # Write debian/control cat >debian/control < Section: misc Priority: optional Standards-Version: $(debian_policy_version) Build-Depends: debhelper (>= 9) Package: $pkgname Architecture: all Depends: \${shlibs:Depends}, \${misc:Depends}, python3 (>= 3.3), python3-urllib3, python3-cryptography, python3-bs4, python3-requests Description: Powerful command-line bookmark manager See https://github.com/jarun/Buku#readme. EOF # Write debian/copyright copyright_file=$builddir/$buildsubdirname/debian/copyright cat >debian/copyright < Source: https://github.com/jarun/Buku Files: * Copyright: 2015-2017 Arun Prakash Jana License: GPL-3 This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License version 3 as published by the Free Software Foundation. . This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. . You should have received a copy of the GNU General Public License along with this program. If not, see debian/source/format <<'EOF' 3.0 (quilt) EOF # Build binary package debuild -us -uc # Copying deb to dist binary_package=$builddir/${pkgname}_${version}-${debrevision}_all.deb cp -p $binary_package $distdir
''' % (timestamp, timestamp)) for row in resultset: out = '%s
') outfp.close() print('%s exported' % count) return True def importdb(self, filepath): '''Import bookmarks from a html or a markdown file (with extension '.md'). Supports Firefox, Google Chrome and IE exported html :param filepath: path to file to import :return: True on success, False on failure ''' if filepath.endswith('.md'): with open(filepath, mode='r', encoding='utf-8') as infp: for line in infp: # Supported markdown format: [title](url) # Find position of title end, url start delimiter combo index = line.find('](') if index != -1: # Find title start delimiter title_start_delim = line[:index].find('[') # Reverse find the url end delimiter url_end_delim = line[index + 2:].rfind(')') if title_start_delim != -1 and url_end_delim > 0: # Parse title title = line[title_start_delim + 1:index] # Parse url url = line[index + 2:index + 2 + url_end_delim] self.add_rec(url, title, None, None, 0, True) self.conn.commit() infp.close() else: try: import bs4 with open(filepath, mode='r', encoding='utf-8') as infp: soup = bs4.BeautifulSoup(infp, 'html.parser') except ImportError: logerr('Beautiful Soup not found') return False except Exception as e: logerr(e) return False html_tags = soup.findAll('a') for tag in html_tags: # Extract comment from