python-slugify-1.2.1/0000755000175000017500000000000012760063500012615 5ustar hlehlepython-slugify-1.2.1/setup.py0000755000175000017500000000517612760063500014343 0ustar hlehle#!/usr/bin/env python # -*- coding: utf-8 -*- from setuptools import setup import re import os import sys import codecs name = 'python-slugify' package = 'slugify' description = 'A Python Slugify application that handles Unicode' url = 'https://github.com/un33k/python-slugify' author = 'Val Neekman' author_email = 'info@neekware.com' license = 'MIT' install_requires = ['Unidecode>=0.04.16'] classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: POSIX', 'Programming Language :: Python', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', ] def get_version(package): """ Return package version as listed in `__version__` in `init.py`. """ init_py = codecs.open(os.path.join(package, '__init__.py'), encoding='utf-8').read() return re.search("^__version__ = ['\"]([^'\"]+)['\"]", init_py, re.MULTILINE).group(1) def get_packages(package): """ Return root package and all sub-packages. """ return [dirpath for dirpath, dirnames, filenames in os.walk(package) if os.path.exists(os.path.join(dirpath, '__init__.py'))] def get_package_data(package): """ Return all files under the root package, that are not in a package themselves. """ walk = [(dirpath.replace(package + os.sep, '', 1), filenames) for dirpath, dirnames, filenames in os.walk(package) if not os.path.exists(os.path.join(dirpath, '__init__.py'))] filepaths = [] for base, filenames in walk: filepaths.extend([os.path.join(base, filename) for filename in filenames]) return {package: filepaths} if sys.argv[-1] == 'publish': os.system("python setup.py sdist upload") args = {'version': get_version(package)} print("You probably want to also tag the version now:") print(" git tag -a %(version)s -m 'version %(version)s' && git push --tags" % args) sys.exit() setup( name=name, version=get_version(package), url=url, license=license, description=description, author=author, author_email=author_email, packages=get_packages(package), package_data=get_package_data(package), install_requires=install_requires, classifiers=classifiers, entry_points={'console_scripts': ['slugify=slugify.slugify:main']}, ) python-slugify-1.2.1/pep8.sh0000755000175000017500000000052512760063500014032 0ustar hlehle#!/bin/bash # Ignoring autogenerated files # -- Migration directories # Ignoring error codes # -- E128 continuation line under-indented for visual indent # -- E261 at least two spaces before inline comment # -- E225 missing whitespace around operator # -- E501 line too long pep8 --ignore=E128,E261,E225,E501 slugify test.py setup.py python-slugify-1.2.1/slugify/0000755000175000017500000000000012760063500014277 5ustar hlehlepython-slugify-1.2.1/slugify/slugify.py0000644000175000017500000001100512760063500016330 0ustar hlehleimport re import unicodedata import types import sys try: from htmlentitydefs import name2codepoint _unicode = unicode _unicode_type = types.UnicodeType except ImportError: from html.entities import name2codepoint _unicode = str _unicode_type = str unichr = chr import unidecode __all__ = ['slugify', 'smart_truncate'] CHAR_ENTITY_PATTERN = re.compile('&(%s);' % '|'.join(name2codepoint)) DECIMAL_PATTERN = re.compile('&#(\d+);') HEX_PATTERN = re.compile('&#x([\da-fA-F]+);') QUOTE_PATTERN = re.compile(r'[\']+') ALLOWED_CHARS_PATTERN = re.compile(r'[^-a-z0-9]+') DUPLICATE_DASH_PATTERN = re.compile('-{2,}') NUMBERS_PATTERN = re.compile('(?<=\d),(?=\d)') def smart_truncate(string, max_length=0, word_boundaries=False, separator=' ', save_order=False): """ Truncate a string. :param string (str): string for modification :param max_length (int): output string length :param word_boundaries (bool): :param save_order (bool): if True then word order of output string is like input string :param separator (str): separator between words :return: """ string = string.strip(separator) if not max_length: return string if len(string) < max_length: return string if not word_boundaries: return string[:max_length].strip(separator) if separator not in string: return string[:max_length] truncated = '' for word in string.split(separator): if word: next_len = len(truncated) + len(word) if next_len < max_length: truncated += '{0}{1}'.format(word, separator) elif next_len == max_length: truncated += '{0}'.format(word) break else: if save_order: break if not truncated: # pragma: no cover truncated = string[:max_length] return truncated.strip(separator) def slugify(text, entities=True, decimal=True, hexadecimal=True, max_length=0, word_boundary=False, separator='-', save_order=False, stopwords=()): """ Make a slug from the given text. :param text (str): initial text :param entities (bool): :param decimal (bool): :param hexadecimal (bool): :param max_length (int): output string length :param word_boundary (bool): :param save_order (bool): if parameter is True and max_length > 0 return whole words in the initial order :param separator (str): separator between words :param stopwords (iterable): words to discount :return (str): """ # ensure text is unicode if not isinstance(text, _unicode_type): text = _unicode(text, 'utf-8', 'ignore') # replace quotes with dashes - pre-process text = QUOTE_PATTERN.sub('-', text) # decode unicode text = unidecode.unidecode(text) # ensure text is still in unicode if not isinstance(text, _unicode_type): text = _unicode(text, 'utf-8', 'ignore') # character entity reference if entities: text = CHAR_ENTITY_PATTERN.sub(lambda m: unichr(name2codepoint[m.group(1)]), text) # decimal character reference if decimal: try: text = DECIMAL_PATTERN.sub(lambda m: unichr(int(m.group(1))), text) except: pass # hexadecimal character reference if hexadecimal: try: text = HEX_PATTERN.sub(lambda m: unichr(int(m.group(1), 16)), text) except: pass # translate text = unicodedata.normalize('NFKD', text) if sys.version_info < (3,): text = text.encode('ascii', 'ignore') # make the text lowercase text = text.lower() # remove generated quotes -- post-process text = QUOTE_PATTERN.sub('', text) # replace unwanted characters text = NUMBERS_PATTERN.sub('', text) text = ALLOWED_CHARS_PATTERN.sub('-', text) # remove redundant - text = DUPLICATE_DASH_PATTERN.sub('-', text).strip('-') # remove stopwords if stopwords: stopwords_lower = [s.lower() for s in stopwords] words = [w for w in text.split('-') if w not in stopwords_lower] text = '-'.join(words) # smart truncate if requested if max_length > 0: text = smart_truncate(text, max_length, word_boundary, '-', save_order) if separator != '-': text = text.replace('-', separator) return text def main(): # pragma: no cover if len(sys.argv) < 2: print("Usage %s TEXT TO SLUGIFY" % sys.argv[0]) else: text = ' '.join(sys.argv[1:]) print(slugify(text)) python-slugify-1.2.1/slugify/__init__.py0000644000175000017500000000026112760063500016407 0ustar hlehlefrom .slugify import * __author__ = 'Val Neekman @ Neekware Inc. [@vneekman]' __description__ = 'A Python slugify application that also handles Unicode' __version__ = '1.2.1' python-slugify-1.2.1/.travis.yml0000644000175000017500000000075112760063500014731 0ustar hlehlesudo: false language: python python: - "2.6" - "2.7" - "3.3" - "3.4" - "3.5" - pypy install: - pip install pip -U - pip install -q -r requirements.txt - pip install -e . - pip install pep8 - pip install coveralls - pip install https://github.com/un33k/pyflakes/tarball/master before_script: - "bash pep8.sh" - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pyflakes -x W slugify; fi script: coverage run --source=slugify test.py after_success: coveralls python-slugify-1.2.1/CHANGELOG.md0000644000175000017500000000332712760063500014433 0ustar hlehle## 1.2.1 - Including certain files (e.g. license.md) in sdists via MANIFEST.in (@proinsias) - Relax licensing by moving from BSD to MIT - Add Python 3.5 support - Add more tests ## 1.2.0 Backward incompatible change: (@fabiocaccamo) - In version < 1.2.0 all single quotes ( ' ) were removed, and moving forward, >= 1.2.0, they will be replaced with ( - ). Example: < 1.2.0 -- ('C\'est déjà l\'été.' -> "cest-deja-lete") >= 1.2.0 -- ('C\'est déjà l\'été.' -> "c-est-deja-l-ete") ## 1.1.4 Bugfix: - Add more test cases, dropped `official` support for python 3.2 ## 1.1.3 Bugfix: - Handle unichar in python 3.x ## 1.1.2 Enhancement: - Ability to remove `stopwords` from string ## 1.0.2 Enhancement: - A new PyPI release ## 1.0.1 Enhancement: - Promoting to production grade ## 0.1.1 Enhancement: - Added option to save word order - Removed 2to3 dependency - Added more tests ## 0.1.0 Enhancement: - Added more test - Added test for python 3.4 ## 0.0.9 Enhancement: - Enable console_scripts ## 0.0.8 Enhancement: - Move logic out of __init__.py - Added console_scripts (@ekamil) - Updated pep8.sh - Added pypy support ## 0.0.7 Enhancement: - Handle encoding in setup file - Update ReadME, ChangeLog, License files ## 0.0.6 Enhancement: - Update for smart_truncate ## 0.0.5 Features: - Added Python 3.2 and 3.3 support (work by: arthurdarcet@github) ## 0.0.4 Features: - Added option to choose non-dash separators (request by: danilodimoia@github) ## 0.0.3 Features: - Added the ability to truncate slugs (request by: juanriaza@github) ## 0.0.2 Enhancement: - Incremental update ## 0.0.1 - Initial version python-slugify-1.2.1/test.py0000644000175000017500000001372012760063500014151 0ustar hlehle# -*- coding: utf-8 -*- import unittest from slugify import slugify from slugify import smart_truncate class TestSlugification(unittest.TestCase): def test_extraneous_seperators(self): txt = "This is a test ---" r = slugify(txt) self.assertEqual(r, "this-is-a-test") txt = "___This is a test ---" r = slugify(txt) self.assertEqual(r, "this-is-a-test") txt = "___This is a test___" r = slugify(txt) self.assertEqual(r, "this-is-a-test") def test_non_word_characters(self): txt = "This -- is a ## test ---" r = slugify(txt) self.assertEqual(r, "this-is-a-test") def test_phonetic_conversion_of_eastern_scripts(self): txt = '影師嗎' r = slugify(txt) self.assertEqual(r, "ying-shi-ma") def test_accented_text(self): txt = 'C\'est déjà l\'été.' r = slugify(txt) self.assertEqual(r, "c-est-deja-l-ete") txt = 'Nín hǎo. Wǒ shì zhōng guó rén' r = slugify(txt) self.assertEqual(r, "nin-hao-wo-shi-zhong-guo-ren") def test_accented_text_with_non_word_characters(self): txt = 'jaja---lol-méméméoo--a' r = slugify(txt) self.assertEqual(r, "jaja-lol-mememeoo-a") def test_cyrillic_text(self): txt = 'Компьютер' r = slugify(txt) self.assertEqual(r, "kompiuter") def test_max_length(self): txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=9) self.assertEqual(r, "jaja-lol") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=15) self.assertEqual(r, "jaja-lol-mememe") def test_max_length_cutoff_not_required(self): txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=50) self.assertEqual(r, "jaja-lol-mememeoo-a") def test_word_boundary(self): txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=15, word_boundary=True) self.assertEqual(r, "jaja-lol-a") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=17, word_boundary=True) self.assertEqual(r, "jaja-lol-mememeoo") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=18, word_boundary=True) self.assertEqual(r, "jaja-lol-mememeoo") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=19, word_boundary=True) self.assertEqual(r, "jaja-lol-mememeoo-a") def test_custom_separator(self): txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=20, word_boundary=True, separator=".") self.assertEqual(r, "jaja.lol.mememeoo.a") def test_multi_character_separator(self): txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=20, word_boundary=True, separator="ZZZZZZ") self.assertEqual(r, "jajaZZZZZZlolZZZZZZmememeooZZZZZZa") def test_save_order(self): txt = 'one two three four five' r = slugify(txt, max_length=13, word_boundary=True, save_order=True) self.assertEqual(r, "one-two-three") txt = 'one two three four five' r = slugify(txt, max_length=13, word_boundary=True, save_order=False) self.assertEqual(r, "one-two-three") txt = 'one two three four five' r = slugify(txt, max_length=12, word_boundary=True, save_order=False) self.assertEqual(r, "one-two-four") txt = 'one two three four five' r = slugify(txt, max_length=12, word_boundary=True, save_order=True) self.assertEqual(r, "one-two") def test_stopword_removal(self): txt = 'this has a stopword' r = slugify(txt, stopwords=['stopword']) self.assertEqual(r, 'this-has-a') def test_multiple_stopword_occurances(self): txt = 'the quick brown fox jumps over the lazy dog' r = slugify(txt, stopwords=['the']) self.assertEqual(r, 'quick-brown-fox-jumps-over-lazy-dog') def test_differently_cased_stopword_match(self): txt = 'Foo A FOO B foo C' r = slugify(txt, stopwords=['foo']) self.assertEqual(r, 'a-b-c') txt = 'Foo A FOO B foo C' r = slugify(txt, stopwords=['FOO']) self.assertEqual(r, 'a-b-c') def test_multiple_stopwords(self): txt = 'the quick brown fox jumps over the lazy dog in a hurry' r = slugify(txt, stopwords=['the', 'in', 'a', 'hurry']) self.assertEqual(r, 'quick-brown-fox-jumps-over-lazy-dog') def test_stopwords_with_different_separator(self): txt = 'the quick brown fox jumps over the lazy dog' r = slugify(txt, stopwords=['the'], separator=' ') self.assertEqual(r, 'quick brown fox jumps over lazy dog') def test_html_entities(self): txt = 'foo & bar' r = slugify(txt) self.assertEqual(r, 'foo-bar') def test_starts_with_number(self): txt = '10 amazing secrets' r = slugify(txt) self.assertEqual(r, '10-amazing-secrets') def test_contains_numbers(self): txt = 'buildings with 1000 windows' r = slugify(txt) self.assertEqual(r, 'buildings-with-1000-windows') def test_ends_with_number(self): txt = 'recipe number 3' r = slugify(txt) self.assertEqual(r, 'recipe-number-3') def test_numbers_only(self): txt = '404' r = slugify(txt) self.assertEqual(r, '404') def test_numbers_and_symbols(self): txt = '1,000 reasons you are #1' r = slugify(txt) self.assertEqual(r, '1000-reasons-you-are-1') class TestUtils(unittest.TestCase): def test_smart_truncate_no_max_length(self): txt = '1,000 reasons you are #1' r = smart_truncate(txt) self.assertEqual(r, txt) def test_smart_truncate_no_seperator(self): txt = '1,000 reasons you are #1' r = smart_truncate(txt, max_length=100, separator='_') self.assertEqual(r, txt) if __name__ == '__main__': unittest.main() python-slugify-1.2.1/requirements.txt0000644000175000017500000000002312760063500016074 0ustar hlehleUnidecode>=0.04.16 python-slugify-1.2.1/README.md0000644000175000017500000001217112760063500014076 0ustar hlehlePython Slugify =============== **A Python slugify application that handles unicode** [![status-image]][status-link] [![version-image]][version-link] [![coverage-image]][coverage-link] Overview ======== A Python **slugify** application that handles **unicode**. How to install ============== 1. easy_install python-slugify 2. pip install python-slugify 3. git clone http://github.com/un33k/python-slugify a. cd python-slugify b. run python setup.py 4. wget https://github.com/un33k/python-slugify/zipball/master a. unzip the downloaded file b. cd into python-slugify-* directory c. run python setup.py How to use =========== ```python from slugify import slugify txt = "This is a test ---" r = slugify(txt) self.assertEqual(r, "this-is-a-test") txt = "___This is a test ---" r = slugify(txt) self.assertEqual(r, "this-is-a-test") txt = "___This is a test___" r = slugify(txt) self.assertEqual(r, "this-is-a-test") txt = "This -- is a ## test ---" r = slugify(txt) self.assertEqual(r, "this-is-a-test") txt = '影師嗎' r = slugify(txt) self.assertEqual(r, "ying-shi-ma") txt = 'C\'est déjà l\'été.' r = slugify(txt) self.assertEqual(r, "c-est-deja-l-ete") txt = 'Nín hǎo. Wǒ shì zhōng guó rén' r = slugify(txt) self.assertEqual(r, "nin-hao-wo-shi-zhong-guo-ren") txt = 'jaja---lol-méméméoo--a' r = slugify(txt) self.assertEqual(r, "jaja-lol-mememeoo-a") txt = 'Компьютер' r = slugify(txt) self.assertEqual(r, "kompiuter") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=9) self.assertEqual(r, "jaja-lol") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=15) self.assertEqual(r, "jaja-lol-mememe") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=50) self.assertEqual(r, "jaja-lol-mememeoo-a") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=15, word_boundary=True) self.assertEqual(r, "jaja-lol-a") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=17, word_boundary=True) self.assertEqual(r, "jaja-lol-mememeoo") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=18, word_boundary=True) self.assertEqual(r, "jaja-lol-mememeoo") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=19, word_boundary=True) self.assertEqual(r, "jaja-lol-mememeoo-a") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=20, word_boundary=True, separator=".") self.assertEqual(r, "jaja.lol.mememeoo.a") txt = 'jaja---lol-méméméoo--a' r = slugify(txt, max_length=20, word_boundary=True, separator="ZZZZZZ") self.assertEqual(r, "jajaZZZZZZlolZZZZZZmememeooZZZZZZa") txt = 'one two three four five' r = slugify(txt, max_length=13, word_boundary=True, save_order=True) self.assertEqual(r, "one-two-three") txt = 'one two three four five' r = slugify(txt, max_length=13, word_boundary=True, save_order=False) self.assertEqual(r, "one-two-three") txt = 'one two three four five' r = slugify(txt, max_length=12, word_boundary=True, save_order=False) self.assertEqual(r, "one-two-four") txt = 'one two three four five' r = slugify(txt, max_length=12, word_boundary=True, save_order=True) self.assertEqual(r, "one-two") txt = 'this has a stopword' r = slugify(txt, stopwords=['stopword']) self.assertEqual(r, 'this-has-a') txt = 'the quick brown fox jumps over the lazy dog' r = slugify(txt, stopwords=['the']) self.assertEqual(r, 'quick-brown-fox-jumps-over-lazy-dog') txt = 'Foo A FOO B foo C' r = slugify(txt, stopwords=['foo']) self.assertEqual(r, 'a-b-c') txt = 'Foo A FOO B foo C' r = slugify(txt, stopwords=['FOO']) self.assertEqual(r, 'a-b-c') txt = 'the quick brown fox jumps over the lazy dog in a hurry' r = slugify(txt, stopwords=['the', 'in', 'a', 'hurry']) self.assertEqual(r, 'quick-brown-fox-jumps-over-lazy-dog') txt = 'foo & bar' r = slugify(txt) self.assertEqual(r, 'foo-bar') ``` Running the tests ================= To run the tests against the current environment: python test.py License ==================== Released under a ([BSD](LICENSE.md)) license. Version ==================== X.Y.Z Version `MAJOR` version -- when you make incompatible API changes, `MINOR` version -- when you add functionality in a backwards-compatible manner, and `PATCH` version -- when you make backwards-compatible bug fixes. [status-image]: https://secure.travis-ci.org/un33k/python-slugify.png?branch=master [status-link]: http://travis-ci.org/un33k/python-slugify?branch=master [version-image]: https://img.shields.io/pypi/v/python-slugify.svg [version-link]: https://pypi.python.org/pypi/python-slugify [coverage-image]: https://coveralls.io/repos/un33k/python-slugify/badge.svg [coverage-link]: https://coveralls.io/r/un33k/python-slugify [download-image]: https://img.shields.io/pypi/dm/python-slugify.svg [download-link]: https://pypi.python.org/pypi/python-slugify python-slugify-1.2.1/LICENSE0000644000175000017500000000211712760063500013623 0ustar hlehleThe MIT License Copyright (c) Val Neekman @ Neekware Inc. http://neekware.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. python-slugify-1.2.1/MANIFEST.in0000644000175000017500000000007312760063500014353 0ustar hlehleinclude LICENSE include README.md include requirements.txt python-slugify-1.2.1/.gitignore0000644000175000017500000000127612760063500014613 0ustar hlehle# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover # Translations *.mo *.pot # Django stuff: *.log # Sphinx documentation docs/_build/ # PyBuilder target/