MechanicalSoup-0.6.0/0000755000000000000000000000000013030510154013076 5ustar rootrootMechanicalSoup-0.6.0/MechanicalSoup.egg-info/0000755000000000000000000000000013030510154017463 5ustar rootrootMechanicalSoup-0.6.0/MechanicalSoup.egg-info/PKG-INFO0000666000000000000000000000133712775647104020615 0ustar rootrootMetadata-Version: 1.1 Name: MechanicalSoup Version: 0.6.0 Summary: A Python library for automating interaction with websites Home-page: https://github.com/hickford/MechanicalSoup Author: UNKNOWN Author-email: UNKNOWN License: MIT Description: UNKNOWN Platform: UNKNOWN Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 MechanicalSoup-0.6.0/MechanicalSoup.egg-info/dependency_links.txt0000666000000000000000000000000112775647104023562 0ustar rootroot MechanicalSoup-0.6.0/MechanicalSoup.egg-info/SOURCES.txt0000666000000000000000000000043312775647104021400 0ustar rootrootsetup.cfg setup.py MechanicalSoup.egg-info/PKG-INFO MechanicalSoup.egg-info/SOURCES.txt MechanicalSoup.egg-info/dependency_links.txt MechanicalSoup.egg-info/requires.txt MechanicalSoup.egg-info/top_level.txt mechanicalsoup/__init__.py mechanicalsoup/browser.py mechanicalsoup/form.pyMechanicalSoup-0.6.0/MechanicalSoup.egg-info/requires.txt0000666000000000000000000000005212775647104022111 0ustar rootrootrequests >= 2.0 beautifulsoup4 six >= 1.4 MechanicalSoup-0.6.0/MechanicalSoup.egg-info/top_level.txt0000666000000000000000000000001712775647104022244 0ustar rootrootmechanicalsoup MechanicalSoup-0.6.0/PKG-INFO0000666000000000000000000000133712775647104014230 0ustar rootrootMetadata-Version: 1.1 Name: MechanicalSoup Version: 0.6.0 Summary: A Python library for automating interaction with websites Home-page: https://github.com/hickford/MechanicalSoup Author: UNKNOWN Author-email: UNKNOWN License: MIT Description: UNKNOWN Platform: UNKNOWN Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 MechanicalSoup-0.6.0/setup.py0000666000000000000000000000277312775646770014662 0ustar rootrootfrom setuptools import setup, find_packages # Always prefer setuptools over distutils from codecs import open # To use a consistent encoding from os import path here = path.abspath(path.dirname(__file__)) setup( name='MechanicalSoup', # useful: python setup.py sdist bdist_wheel upload version='0.6.0', description='A Python library for automating interaction with websites', url='https://github.com/hickford/MechanicalSoup', license='MIT', classifiers=[ 'License :: OSI Approved :: MIT License', # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', ], packages=['mechanicalsoup'], # List run-time dependencies here. These will be installed by pip when your # project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=[ 'requests >= 2.0', 'beautifulsoup4', 'six >= 1.4' ], ) MechanicalSoup-0.6.0/mechanicalsoup/0000755000000000000000000000000013030510154016071 5ustar rootrootMechanicalSoup-0.6.0/mechanicalsoup/__init__.py0000666000000000000000000000006412551023406020214 0ustar rootrootfrom .browser import Browser from .form import FormMechanicalSoup-0.6.0/mechanicalsoup/form.py0000666000000000000000000000320212714560342017423 0ustar rootrootclass Form(object): def __init__(self, form): self.form = form def input(self, data): for (name, value) in data.items(): self.form.find("input", {"name": name})["value"] = value def check(self, data): for (name, value) in data.items(): if isinstance(value, list): for choice in value: self.form.find("input", {"name": name, "value": choice})[ "checked"] = "" else: self.form.find("input", {"name": name, "value": value})[ "checked"] = "" def textarea(self, data): for (name, value) in data.items(): self.form.find("textarea", {"name": name}).insert(0, value) def attach(self, data): for (name, value) in data.items(): self.form.find("input", {"name": name})["value"] = value def choose_submit(self, el): # In a normal web browser, when a input[type=submit] is clicked, # all other submits aren't sent. You can use simulate this as following: # page = browser.get(URL) # form_el = page.soup.form # form = Form(form_el) # submit = page.soup.select(SUBMIT_SELECTOR)[0] # form.choose_submit(submit) # url = BASE_DOMAIN + form_el.attrs['action'] # return browser.submit(form, url) for inp in self.form.select("input"): if inp.get('type') != 'submit': continue if inp == el: continue del inp['name'] return True return False MechanicalSoup-0.6.0/mechanicalsoup/browser.py0000666000000000000000000001060612775646634020171 0ustar rootrootimport warnings import requests import bs4 from six.moves import urllib from six import string_types from .form import Form # see https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use warnings.filterwarnings("ignore", "No parser was explicitly specified", module="bs4") class Browser(object): def __init__(self, session=None, soup_config=None, requests_adapters=None): self.session = session or requests.Session() if requests_adapters is not None: for adaptee, adapter in requests_adapters.items(): self.session.mount(adaptee, adapter) self.soup_config = soup_config or dict() @staticmethod def add_soup(response, soup_config): if "text/html" in response.headers.get("Content-Type", ""): response.soup = bs4.BeautifulSoup( response.content, **soup_config) def request(self, *args, **kwargs): response = self.session.request(*args, **kwargs) Browser.add_soup(response, self.soup_config) return response def get(self, *args, **kwargs): response = self.session.get(*args, **kwargs) Browser.add_soup(response, self.soup_config) return response def post(self, *args, **kwargs): response = self.session.post(*args, **kwargs) Browser.add_soup(response, self.soup_config) return response def _build_request(self, form, url=None, **kwargs): method = form.get("method", "get") action = form.get("action") url = urllib.parse.urljoin(url, action) if url is None: # This happens when both `action` and `url` are None. raise ValueError('no URL to submit to') # read http://www.w3.org/TR/html5/forms.html data = kwargs.pop("data", dict()) files = kwargs.pop("files", dict()) for input in form.select("input"): name = input.get("name") if not name: continue if input.get("type") in ("radio", "checkbox"): if "checked" not in input.attrs: continue value = input.get("value", "on") else: # web browsers use empty string for inputs with missing values value = input.get("value", "") if input.get("type") == "checkbox": data.setdefault(name, []).append(value) elif input.get("type") == "file": # read http://www.cs.tut.fi/~jkorpela/forms/file.html # in web browsers, file upload only happens if the form"s (or # submit button"s) enctype attribute is set to # "multipart/form-data". we don"t care, simplify. if not value: continue if isinstance(value, string_types): value = open(value, "rb") files[name] = value else: data[name] = value for textarea in form.select("textarea"): name = textarea.get("name") if not name: continue data[name] = textarea.text for select in form.select("select"): name = select.get("name") if not name: continue multiple = "multiple" in select.attrs values = [] for i, option in enumerate(select.select("option")): if (i == 0 and not multiple) or "selected" in option.attrs: values.append(option.get("value", "")) if multiple: data[name] = values elif values: data[name] = values[-1] if method.lower() == "get": kwargs["params"] = data else: kwargs["data"] = data return requests.Request(method, url, files=files, **kwargs) def _prepare_request(self, form, url=None, **kwargs): request = self._build_request(form, url, **kwargs) return self.session.prepare_request(request) def submit(self, form, url=None, **kwargs): if isinstance(form, Form): form = form.form request = self._prepare_request(form, url, **kwargs) response = self.session.send(request) Browser.add_soup(response, self.soup_config) return response MechanicalSoup-0.6.0/setup.cfg0000666000000000000000000000014012775647104014743 0ustar rootroot[bdist_wheel] universal = 1 [egg_info] tag_build = tag_date = 0 tag_svn_revision = 0