python-confluent-kafka-0.9.2/000077500000000000000000000000001303570112400161055ustar00rootroot00000000000000python-confluent-kafka-0.9.2/LICENSE000066400000000000000000000260751303570112400171240ustar00rootroot00000000000000Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. python-confluent-kafka-0.9.2/MANIFEST.in000066400000000000000000000001051303570112400176370ustar00rootroot00000000000000include README.md include LICENSE include confluent_kafka/src/*.[ch] python-confluent-kafka-0.9.2/PKG-INFO000066400000000000000000000004421303570112400172020ustar00rootroot00000000000000Metadata-Version: 1.0 Name: confluent-kafka Version: 0.9.2 Summary: Confluent's Apache Kafka client for Python Home-page: https://github.com/confluentinc/confluent-kafka-python Author: Confluent Inc Author-email: support@confluent.io License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN python-confluent-kafka-0.9.2/README.md000066400000000000000000000063451303570112400173740ustar00rootroot00000000000000Confluent's Apache Kafka client for Python ========================================== Confluent's Kafka client for Python wraps the librdkafka C library, providing full Kafka protocol support with great performance and reliability. The Python bindings provides a high-level Producer and Consumer with support for the balanced consumer groups of Apache Kafka 0.9. See the [API documentation](http://docs.confluent.io/current/clients/confluent-kafka-python/index.html) for more info. **License**: [Apache License v2.0](http://www.apache.org/licenses/LICENSE-2.0) Usage ===== **Producer:** ```python from confluent_kafka import Producer p = Producer({'bootstrap.servers': 'mybroker,mybroker2'}) for data in some_data_source: p.produce('mytopic', data.encode('utf-8')) p.flush() ``` **High-level Consumer:** ```python from confluent_kafka import Consumer, KafkaError c = Consumer({'bootstrap.servers': 'mybroker', 'group.id': 'mygroup', 'default.topic.config': {'auto.offset.reset': 'smallest'}}) c.subscribe(['mytopic']) running = True while running: msg = c.poll() if not msg.error(): print('Received message: %s' % msg.value().decode('utf-8')) elif msg.error().code() != KafkaError._PARTITION_EOF: print(msg.error()) running = False c.close() ``` See [examples](examples) for more examples. Broker compatibility ==================== The Python client (as well as the underlying C library librdkafka) supports all broker versions >= 0.8. But due to the nature of the Kafka protocol in broker versions 0.8 and 0.9 it is not safe for a client to assume what protocol version is actually supported by the broker, thus you will need to hint the Python client what protocol version it may use. This is done through two configuration settings: * `broker.version.fallback=YOUR_BROKER_VERSION` (default 0.9.0.1) * `api.version.request=true|false` (default false) When using a Kafka 0.10 broker or later you only need to set `api.version.request=true`. If you use Kafka broker 0.9 or 0.8 you should leave `api.version.request=false` (default) and set `broker.version.fallback` to your broker version, e.g `broker.version.fallback=0.9.0.1`. More info here: https://github.com/edenhill/librdkafka/wiki/Broker-version-compatibility Prerequisites ============= * Python >= 2.7 or Python 3.x * [librdkafka](https://github.com/edenhill/librdkafka) >= 0.9.1 Install ======= **Install from PyPi:** $ pip install confluent-kafka **Install from source / tarball:** $ pip install . Build ===== $ python setup.py build If librdkafka is installed in a non-standard location provide the include and library directories with: $ C_INCLUDE_PATH=/path/to/include LIBRARY_PATH=/path/to/lib python setup.py ... Tests ===== **Run unit-tests:** $ py.test **NOTE**: Requires `py.test`, install by `pip install pytest` **Run integration tests:** $ examples/integration_test.py **WARNING**: These tests require an active Kafka cluster and will make use of a topic named 'test'. Generate documentation ====================== Install sphinx and sphinx_rtd_theme packages and then: $ make docs or: $ python setup.py build_sphinx Documentation will be generated in `docs/_build/`. python-confluent-kafka-0.9.2/confluent_kafka.egg-info/000077500000000000000000000000001303570112400227315ustar00rootroot00000000000000python-confluent-kafka-0.9.2/confluent_kafka.egg-info/PKG-INFO000066400000000000000000000004421303570112400240260ustar00rootroot00000000000000Metadata-Version: 1.0 Name: confluent-kafka Version: 0.9.2 Summary: Confluent's Apache Kafka client for Python Home-page: https://github.com/confluentinc/confluent-kafka-python Author: Confluent Inc Author-email: support@confluent.io License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN python-confluent-kafka-0.9.2/confluent_kafka.egg-info/SOURCES.txt000066400000000000000000000013231303570112400246140ustar00rootroot00000000000000LICENSE MANIFEST.in README.md setup.py confluent_kafka/__init__.py confluent_kafka.egg-info/PKG-INFO confluent_kafka.egg-info/SOURCES.txt confluent_kafka.egg-info/dependency_links.txt confluent_kafka.egg-info/top_level.txt confluent_kafka/kafkatest/__init__.py confluent_kafka/kafkatest/verifiable_client.py confluent_kafka/kafkatest/verifiable_consumer.py confluent_kafka/kafkatest/verifiable_producer.py confluent_kafka/src/Consumer.c confluent_kafka/src/Producer.c confluent_kafka/src/confluent_kafka.c confluent_kafka/src/confluent_kafka.h tests/test_Consumer.py tests/test_KafkaError.py tests/test_Producer.py tests/test_TopicPartition.py tests/test_docs.py tests/test_enums.py tests/test_misc.py tests/test_threads.pypython-confluent-kafka-0.9.2/confluent_kafka.egg-info/dependency_links.txt000066400000000000000000000000011303570112400267770ustar00rootroot00000000000000 python-confluent-kafka-0.9.2/confluent_kafka.egg-info/top_level.txt000066400000000000000000000000201303570112400254530ustar00rootroot00000000000000confluent_kafka python-confluent-kafka-0.9.2/confluent_kafka/000077500000000000000000000000001303570112400212375ustar00rootroot00000000000000python-confluent-kafka-0.9.2/confluent_kafka/__init__.py000066400000000000000000000000651303570112400233510ustar00rootroot00000000000000__all__ = ['cimpl','kafkatest'] from .cimpl import * python-confluent-kafka-0.9.2/confluent_kafka/kafkatest/000077500000000000000000000000001303570112400232145ustar00rootroot00000000000000python-confluent-kafka-0.9.2/confluent_kafka/kafkatest/__init__.py000066400000000000000000000001251303570112400253230ustar00rootroot00000000000000""" Python client implementations of the official Kafka tests/kafkatest clients. """ python-confluent-kafka-0.9.2/confluent_kafka/kafkatest/verifiable_client.py000066400000000000000000000055011303570112400272350ustar00rootroot00000000000000# Copyright 2016 Confluent Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import signal, socket, os, sys, time, json, re, datetime class VerifiableClient(object): """ Generic base class for a kafkatest verifiable client. Implements the common kafkatest protocol and semantics. """ def __init__ (self, conf): """ """ super(VerifiableClient, self).__init__() self.conf = conf self.conf['client.id'] = 'python@' + socket.gethostname() self.run = True signal.signal(signal.SIGTERM, self.sig_term) self.dbg('Pid is %d' % os.getpid()) def sig_term (self, sig, frame): self.dbg('SIGTERM') self.run = False @staticmethod def _timestamp (): return time.strftime('%H:%M:%S', time.localtime()) def dbg (self, s): """ Debugging printout """ sys.stderr.write('%% %s DEBUG: %s\n' % (self._timestamp(), s)) def err (self, s, term=False): """ Error printout, if term=True the process will terminate immediately. """ sys.stderr.write('%% %s ERROR: %s\n' % (self._timestamp(), s)) if term: sys.stderr.write('%% FATAL ERROR ^\n') sys.exit(1) def send (self, d): """ Send dict as JSON to stdout for consumtion by kafkatest handler """ d['_time'] = str(datetime.datetime.now()) self.dbg('SEND: %s' % json.dumps(d)) sys.stdout.write('%s\n' % json.dumps(d)) sys.stdout.flush() @staticmethod def set_config (conf, args): """ Set client config properties using args dict. """ for n,v in args.iteritems(): if v is None: continue # Things to ignore if '.' not in n: # App config, skip continue if n.startswith('topic.'): # Set "topic.<...>" properties on default topic conf dict conf['default.topic.config'][n[6:]] = v elif n == 'partition.assignment.strategy': # Convert Java class name to config value. # "org.apache.kafka.clients.consumer.RangeAssignor" -> "range" conf[n] = re.sub(r'org.apache.kafka.clients.consumer.(\w+)Assignor', lambda x: x.group(1).lower(), v) else: conf[n] = v python-confluent-kafka-0.9.2/confluent_kafka/kafkatest/verifiable_consumer.py000077500000000000000000000236571303570112400276310ustar00rootroot00000000000000#!/usr/bin/env python # # Copyright 2016 Confluent Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import argparse, sys from confluent_kafka import Consumer, KafkaError, KafkaException from verifiable_client import VerifiableClient class VerifiableConsumer(VerifiableClient): """ confluent-kafka-python backed VerifiableConsumer class for use with Kafka's kafkatests client tests. """ def __init__ (self, conf): """ \p conf is a config dict passed to confluent_kafka.Consumer() """ super(VerifiableConsumer, self).__init__(conf) self.conf['on_commit'] = self.on_commit self.consumer = Consumer(**conf) self.consumed_msgs = 0 self.consumed_msgs_last_reported = 0 self.consumed_msgs_at_last_commit = 0 self.use_auto_commit = False self.use_async_commit = False self.max_msgs = -1 self.assignment = [] self.assignment_dict = dict() def find_assignment (self, topic, partition): """ Find and return existing assignment based on \p topic and \p partition, or None on miss. """ skey = '%s %d' % (topic, partition) return self.assignment_dict.get(skey) def send_records_consumed (self, immediate=False): """ Send records_consumed, every 100 messages, on timeout, or if immediate is set. """ if (self.consumed_msgs <= self.consumed_msgs_last_reported + (0 if immediate else 100)): return if len(self.assignment) == 0: return d = {'name': 'records_consumed', 'count': self.consumed_msgs - self.consumed_msgs_last_reported, 'partitions': []} for a in self.assignment: if a.min_offset == -1: # Skip partitions that havent had any messages since last time. # This is to circumvent some minOffset checks in kafkatest. continue d['partitions'].append(a.to_dict()) a.min_offset = -1 self.send(d) self.consumed_msgs_last_reported = self.consumed_msgs def send_assignment (self, evtype, partitions): """ Send assignment update, \p evtype is either 'assigned' or 'revoked' """ d = { 'name': 'partitions_' + evtype, 'partitions': [{'topic': x.topic, 'partition': x.partition} for x in partitions]} self.send(d) def on_assign (self, consumer, partitions): """ Rebalance on_assign callback """ old_assignment = self.assignment self.assignment = [AssignedPartition(p.topic, p.partition) for p in partitions] # Move over our last seen offsets so that we can report a proper # minOffset even after a rebalance loop. for a in old_assignment: b = self.find_assignment(a.topic, a.partition) b.min_offset = a.min_offset self.assignment_dict = {a.skey: a for a in self.assignment} self.send_assignment('assigned', partitions) def on_revoke (self, consumer, partitions): """ Rebalance on_revoke callback """ # Send final consumed records prior to rebalancing to make sure # latest consumed is in par with what is going to be committed. self.send_records_consumed(immediate=True) self.assignment = list() self.assignment_dict = dict() self.send_assignment('revoked', partitions) self.do_commit(immediate=True) def on_commit (self, err, partitions): """ Offsets Committed callback """ if err is not None and err.code() == KafkaError._NO_OFFSET: self.dbg('on_commit(): no offsets to commit') return # Report consumed messages to make sure consumed position >= committed position self.send_records_consumed(immediate=True) d = {'name': 'offsets_committed', 'offsets': []} if err is not None: d['success'] = False d['error'] = str(err) else: d['success'] = True d['error'] = '' for p in partitions: pd = {'topic': p.topic, 'partition': p.partition, 'offset': p.offset, 'error': str(p.error)} d['offsets'].append(pd) self.send(d) def do_commit (self, immediate=False, async=None): """ Commit every 1000 messages or whenever there is a consume timeout or immediate. """ if (self.use_auto_commit or self.consumed_msgs_at_last_commit + (0 if immediate else 1000) > self.consumed_msgs): return # Make sure we report consumption before commit, # otherwise tests may fail because of commit > consumed if self.consumed_msgs_at_last_commit < self.consumed_msgs: self.send_records_consumed(immediate=True) if async is None: async_mode = self.use_async_commit else: async_mode = async self.dbg('Committing %d messages (Async=%s)' % (self.consumed_msgs - self.consumed_msgs_at_last_commit, async_mode)) try: self.consumer.commit(async=async_mode) except KafkaException as e: if e.args[0].code() == KafkaError._WAIT_COORD: self.dbg('Ignoring commit failure, still waiting for coordinator') elif e.args[0].code() == KafkaError._NO_OFFSET: self.dbg('No offsets to commit') else: raise self.consumed_msgs_at_last_commit = self.consumed_msgs def msg_consume (self, msg): """ Handle consumed message (or error event) """ if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: # ignore EOF pass else: self.err('Consume failed: %s' % msg.error(), term=True) return if False: self.dbg('Read msg from %s [%d] @ %d' % \ (msg.topic(), msg.partition(), msg.offset())) if self.max_msgs >= 0 and self.consumed_msgs >= self.max_msgs: return # ignore extra messages # Find assignment. a = self.find_assignment(msg.topic(), msg.partition()) if a is None: self.err('Received message on unassigned partition %s [%d] @ %d' % (msg.topic(), msg.partition(), msg.offset()), term=True) a.consumed_msgs += 1 if a.min_offset == -1: a.min_offset = msg.offset() if a.max_offset < msg.offset(): a.max_offset = msg.offset() self.consumed_msgs += 1 self.send_records_consumed(immediate=False) self.do_commit(immediate=False) class AssignedPartition(object): """ Local state container for assigned partition. """ def __init__ (self, topic, partition): super(AssignedPartition, self).__init__() self.topic = topic self.partition = partition self.skey = '%s %d' % (self.topic, self.partition) self.consumed_msgs = 0 self.min_offset = -1 self.max_offset = 0 def to_dict (self): """ Return a dict of this partition's state """ return {'topic': self.topic, 'partition': self.partition, 'minOffset': self.min_offset, 'maxOffset': self.max_offset} if __name__ == '__main__': parser = argparse.ArgumentParser(description='Verifiable Python Consumer') parser.add_argument('--topic', action='append', type=str, required=True) parser.add_argument('--group-id', dest='group.id', required=True) parser.add_argument('--broker-list', dest='bootstrap.servers', required=True) parser.add_argument('--session-timeout', type=int, dest='session.timeout.ms', default=6000) parser.add_argument('--enable-autocommit', action='store_true', dest='enable.auto.commit', default=False) parser.add_argument('--max-messages', type=int, dest='max_messages', default=-1) parser.add_argument('--assignment-strategy', dest='partition.assignment.strategy') parser.add_argument('--reset-policy', dest='topic.auto.offset.reset', default='earliest') parser.add_argument('--consumer.config', dest='consumer_config') args = vars(parser.parse_args()) conf = {'broker.version.fallback': '0.9.0', 'default.topic.config': dict()} VerifiableClient.set_config(conf, args) vc = VerifiableConsumer(conf) vc.use_auto_commit = args['enable.auto.commit'] vc.max_msgs = args['max_messages'] vc.dbg('Using config: %s' % conf) vc.dbg('Subscribing to %s' % args['topic']) vc.consumer.subscribe(args['topic'], on_assign=vc.on_assign, on_revoke=vc.on_revoke) try: while vc.run: msg = vc.consumer.poll(timeout=1.0) if msg is None: # Timeout. # Try reporting consumed messages vc.send_records_consumed(immediate=True) # Commit every poll() timeout instead of on every message. # Also commit on every 1000 messages, whichever comes first. vc.do_commit(immediate=True) continue # Handle message (or error event) vc.msg_consume(msg) except KeyboardInterrupt: pass vc.dbg('Closing consumer') vc.send_records_consumed(immediate=True) if not vc.use_auto_commit: vc.do_commit(immediate=True, async=False) vc.consumer.close() vc.send({'name': 'shutdown_complete'}) vc.dbg('All done') python-confluent-kafka-0.9.2/confluent_kafka/kafkatest/verifiable_producer.py000077500000000000000000000112021303570112400276000ustar00rootroot00000000000000#!/usr/bin/env python # # Copyright 2016 Confluent Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import argparse, time from confluent_kafka import Producer, KafkaError, KafkaException from verifiable_client import VerifiableClient class VerifiableProducer(VerifiableClient): """ confluent-kafka-python backed VerifiableProducer class for use with Kafka's kafkatests client tests. """ def __init__ (self, conf): """ \p conf is a config dict passed to confluent_kafka.Producer() """ super(VerifiableProducer, self).__init__(conf) self.conf['on_delivery'] = self.dr_cb self.conf['default.topic.config']['produce.offset.report'] = True self.producer = Producer(**self.conf) self.num_acked = 0 self.num_sent = 0 self.num_err = 0 def dr_cb (self, err, msg): """ Per-message Delivery report callback. Called from poll() """ if err: self.num_err += 1 self.send({'name': 'producer_send_error', 'message': str(err), 'topic': msg.topic(), 'key': msg.key(), 'value': msg.value()}) else: self.num_acked += 1 self.send({'name': 'producer_send_success', 'topic': msg.topic(), 'partition': msg.partition(), 'offset': msg.offset(), 'key': msg.key(), 'value': msg.value()}) pass if __name__ == '__main__': parser = argparse.ArgumentParser(description='Verifiable Python Producer') parser.add_argument('--topic', type=str, required=True) parser.add_argument('--throughput', type=int, default=0) parser.add_argument('--broker-list', dest='bootstrap.servers', required=True) parser.add_argument('--max-messages', type=int, dest='max_msgs', default=1000000) # avoid infinite parser.add_argument('--value-prefix', dest='value_prefix', type=str, default=None) parser.add_argument('--acks', type=int, dest='topic.request.required.acks', default=-1) parser.add_argument('--producer.config', dest='producer_config') args = vars(parser.parse_args()) conf = {'broker.version.fallback': '0.9.0', 'default.topic.config': dict()} VerifiableClient.set_config(conf, args) vp = VerifiableProducer(conf) vp.max_msgs = args['max_msgs'] throughput = args['throughput'] topic = args['topic'] if args['value_prefix'] is not None: value_fmt = args['value_prefix'] + '.%d' else: value_fmt = '%d' if throughput > 0: delay = 1.0/throughput else: delay = 0 vp.dbg('Producing %d messages at a rate of %d/s' % (vp.max_msgs, throughput)) try: for i in range(0, vp.max_msgs): if not vp.run: break t_end = time.time() + delay while vp.run: try: vp.producer.produce(topic, value=(value_fmt % i)) vp.num_sent += 1 except KafkaException as e: self.err('produce() #%d/%d failed: %s' % \ (i, vp.max_msgs, str(e))) vp.num_err += 1 except BufferError: vp.dbg('Local produce queue full (produced %d/%d msgs), waiting for deliveries..' % \ (i, vp.max_msgs)) vp.producer.poll(timeout=0.5) continue break # Delay to achieve desired throughput, # but make sure poll is called at least once # to serve DRs. while True: remaining = max(0, t_end - time.time()) vp.producer.poll(timeout=remaining) if remaining <= 0.00000001: break except KeyboardInterrupt: pass # Flush remaining messages to broker. vp.dbg('Flushing') try: vp.producer.flush() except KeyboardInterrupt: pass vp.send({'name': 'shutdown_complete'}) vp.dbg('All done') python-confluent-kafka-0.9.2/confluent_kafka/src/000077500000000000000000000000001303570112400220265ustar00rootroot00000000000000python-confluent-kafka-0.9.2/confluent_kafka/src/Consumer.c000066400000000000000000000451071303570112400237740ustar00rootroot00000000000000/** * Copyright 2016 Confluent Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "confluent_kafka.h" /**************************************************************************** * * * Consumer * * * * ****************************************************************************/ static int Consumer_clear (Handle *self) { if (self->u.Consumer.on_assign) { Py_DECREF(self->u.Consumer.on_assign); self->u.Consumer.on_assign = NULL; } if (self->u.Consumer.on_revoke) { Py_DECREF(self->u.Consumer.on_revoke); self->u.Consumer.on_revoke = NULL; } if (self->u.Consumer.on_commit) { Py_DECREF(self->u.Consumer.on_commit); self->u.Consumer.on_commit = NULL; } Handle_clear(self); return 0; } static void Consumer_dealloc (Handle *self) { PyObject_GC_UnTrack(self); Consumer_clear(self); if (self->rk) rd_kafka_destroy(self->rk); Py_TYPE(self)->tp_free((PyObject *)self); } static int Consumer_traverse (Handle *self, visitproc visit, void *arg) { if (self->u.Consumer.on_assign) Py_VISIT(self->u.Consumer.on_assign); if (self->u.Consumer.on_revoke) Py_VISIT(self->u.Consumer.on_revoke); if (self->u.Consumer.on_commit) Py_VISIT(self->u.Consumer.on_commit); Handle_traverse(self, visit, arg); return 0; } static PyObject *Consumer_subscribe (Handle *self, PyObject *args, PyObject *kwargs) { rd_kafka_topic_partition_list_t *topics; static char *kws[] = { "topics", "on_assign", "on_revoke", NULL }; PyObject *tlist, *on_assign = NULL, *on_revoke = NULL; Py_ssize_t pos = 0; rd_kafka_resp_err_t err; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OO", kws, &tlist, &on_assign, &on_revoke)) return NULL; if (!PyList_Check(tlist)) { PyErr_Format(PyExc_TypeError, "expected list of topic unicode strings"); return NULL; } if (on_assign && !PyCallable_Check(on_assign)) { PyErr_Format(PyExc_TypeError, "on_assign expects a callable"); return NULL; } if (on_revoke && !PyCallable_Check(on_revoke)) { PyErr_Format(PyExc_TypeError, "on_revoke expects a callable"); return NULL; } topics = rd_kafka_topic_partition_list_new((int)PyList_Size(tlist)); for (pos = 0 ; pos < PyList_Size(tlist) ; pos++) { PyObject *o = PyList_GetItem(tlist, pos); PyObject *uo; if (!(uo = cfl_PyObject_Unistr(o))) { PyErr_Format(PyExc_TypeError, "expected list of unicode strings"); rd_kafka_topic_partition_list_destroy(topics); return NULL; } rd_kafka_topic_partition_list_add(topics, cfl_PyUnistr_AsUTF8(uo), RD_KAFKA_PARTITION_UA); Py_DECREF(uo); } err = rd_kafka_subscribe(self->rk, topics); rd_kafka_topic_partition_list_destroy(topics); if (err) { cfl_PyErr_Format(err, "Failed to set subscription: %s", rd_kafka_err2str(err)); return NULL; } /* * Update rebalance callbacks */ if (self->u.Consumer.on_assign) { Py_DECREF(self->u.Consumer.on_assign); self->u.Consumer.on_assign = NULL; } if (on_assign) { self->u.Consumer.on_assign = on_assign; Py_INCREF(self->u.Consumer.on_assign); } if (self->u.Consumer.on_revoke) { Py_DECREF(self->u.Consumer.on_revoke); self->u.Consumer.on_revoke = NULL; } if (on_revoke) { self->u.Consumer.on_revoke = on_revoke; Py_INCREF(self->u.Consumer.on_revoke); } Py_RETURN_NONE; } static PyObject *Consumer_unsubscribe (Handle *self, PyObject *ignore) { rd_kafka_resp_err_t err; err = rd_kafka_unsubscribe(self->rk); if (err) { cfl_PyErr_Format(err, "Failed to remove subscription: %s", rd_kafka_err2str(err)); return NULL; } Py_RETURN_NONE; } static PyObject *Consumer_assign (Handle *self, PyObject *tlist) { rd_kafka_topic_partition_list_t *c_parts; rd_kafka_resp_err_t err; if (!(c_parts = py_to_c_parts(tlist))) return NULL; self->u.Consumer.rebalance_assigned++; err = rd_kafka_assign(self->rk, c_parts); rd_kafka_topic_partition_list_destroy(c_parts); if (err) { cfl_PyErr_Format(err, "Failed to set assignment: %s", rd_kafka_err2str(err)); return NULL; } Py_RETURN_NONE; } static PyObject *Consumer_unassign (Handle *self, PyObject *ignore) { rd_kafka_resp_err_t err; self->u.Consumer.rebalance_assigned++; err = rd_kafka_assign(self->rk, NULL); if (err) { cfl_PyErr_Format(err, "Failed to remove assignment: %s", rd_kafka_err2str(err)); return NULL; } Py_RETURN_NONE; } static PyObject *Consumer_commit (Handle *self, PyObject *args, PyObject *kwargs) { rd_kafka_resp_err_t err; PyObject *msg = NULL, *offsets = NULL, *async_o = NULL; rd_kafka_topic_partition_list_t *c_offsets; int async = 1; static char *kws[] = { "message", "offsets", "async",NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OOO", kws, &msg, &offsets, &async_o)) return NULL; if (msg && offsets) { PyErr_SetString(PyExc_ValueError, "message and offsets are mutually exclusive"); return NULL; } if (async_o) async = PyObject_IsTrue(async_o); if (offsets) { if (!(c_offsets = py_to_c_parts(offsets))) return NULL; } else if (msg) { Message *m; if (PyObject_Type((PyObject *)msg) != (PyObject *)&MessageType) { PyErr_Format(PyExc_TypeError, "expected %s", MessageType.tp_name); return NULL; } m = (Message *)msg; c_offsets = rd_kafka_topic_partition_list_new(1); rd_kafka_topic_partition_list_add( c_offsets, cfl_PyUnistr_AsUTF8(m->topic), m->partition)->offset =m->offset + 1; } else { c_offsets = NULL; err = rd_kafka_commit(self->rk, NULL, async); } err = rd_kafka_commit(self->rk, c_offsets, async); if (c_offsets) rd_kafka_topic_partition_list_destroy(c_offsets); if (err) { cfl_PyErr_Format(err, "Commit failed: %s", rd_kafka_err2str(err)); return NULL; } Py_RETURN_NONE; } static PyObject *Consumer_committed (Handle *self, PyObject *args, PyObject *kwargs) { PyObject *plist; rd_kafka_topic_partition_list_t *c_parts; rd_kafka_resp_err_t err; double tmout = -1.0f; static char *kws[] = { "partitions", "timeout", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|d", kws, &plist, &tmout)) return NULL; if (!(c_parts = py_to_c_parts(plist))) return NULL; err = rd_kafka_committed(self->rk, c_parts, tmout >= 0 ? (int)(tmout * 1000.0f) : -1); if (err) { rd_kafka_topic_partition_list_destroy(c_parts); cfl_PyErr_Format(err, "Failed to get committed offsets: %s", rd_kafka_err2str(err)); return NULL; } plist = c_parts_to_py(c_parts); rd_kafka_topic_partition_list_destroy(c_parts); return plist; } static PyObject *Consumer_position (Handle *self, PyObject *args, PyObject *kwargs) { PyObject *plist; rd_kafka_topic_partition_list_t *c_parts; rd_kafka_resp_err_t err; static char *kws[] = { "partitions", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", kws, &plist)) return NULL; if (!(c_parts = py_to_c_parts(plist))) return NULL; err = rd_kafka_position(self->rk, c_parts); if (err) { rd_kafka_topic_partition_list_destroy(c_parts); cfl_PyErr_Format(err, "Failed to get position: %s", rd_kafka_err2str(err)); return NULL; } plist = c_parts_to_py(c_parts); rd_kafka_topic_partition_list_destroy(c_parts); return plist; } static PyObject *Consumer_poll (Handle *self, PyObject *args, PyObject *kwargs) { double tmout = -1.0f; static char *kws[] = { "timeout", NULL }; rd_kafka_message_t *rkm; PyObject *msgobj; CallState cs; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|d", kws, &tmout)) return NULL; CallState_begin(self, &cs); rkm = rd_kafka_consumer_poll(self->rk, tmout >= 0 ? (int)(tmout * 1000.0f) : -1); if (!CallState_end(self, &cs)) return NULL; if (!rkm) Py_RETURN_NONE; msgobj = Message_new0(rkm); rd_kafka_message_destroy(rkm); return msgobj; } static PyObject *Consumer_close (Handle *self, PyObject *ignore) { CallState cs; int raise = 0; CallState_begin(self, &cs); rd_kafka_consumer_close(self->rk); raise = !CallState_end(self, &cs); rd_kafka_destroy(self->rk); self->rk = NULL; if (raise) return NULL; Py_RETURN_NONE; } static PyMethodDef Consumer_methods[] = { { "subscribe", (PyCFunction)Consumer_subscribe, METH_VARARGS|METH_KEYWORDS, ".. py:function:: subscribe(topics, [listener=None])\n" "\n" " Set subscription to supplied list of topics\n" " This replaces a previous subscription.\n" "\n" " :param list(str) topics: List of topics (strings) to subscribe to.\n" " :param callable on_assign: callback to provide handling of " "customized offsets on completion of a successful partition " "re-assignment.\n" " :param callable on_revoke: callback to provide handling of " "offset commits to a customized store on the start of a " "rebalance operation.\n" "\n" " :raises KafkaException:\n" "\n" "\n" ".. py:function:: on_assign(consumer, partitions)\n" ".. py:function:: on_revoke(consumer, partitions)\n" "\n" " :param Consumer consumer: Consumer instance.\n" " :param list(TopicPartition) partitions: Absolute list of partitions being assigned or revoked.\n" "\n" }, { "unsubscribe", (PyCFunction)Consumer_unsubscribe, METH_NOARGS, " Remove current subscription.\n" "\n" }, { "poll", (PyCFunction)Consumer_poll, METH_VARARGS|METH_KEYWORDS, ".. py:function:: poll([timeout=None])\n" "\n" " Consume messages, calls callbacks and returns events.\n" "\n" " The application must check the returned :py:class:`Message` " "object's :py:func:`Message.error()` method to distinguish " "between proper messages (error() returns None), or an event or " "error (see error().code() for specifics).\n" "\n" " .. note: Callbacks may be called from this method, " "such as ``on_assign``, ``on_revoke``, et.al.\n" "\n" " :param float timeout: Maximum time to block waiting for message, event or callback.\n" " :returns: A Message object or None on timeout\n" " :rtype: :py:class:`Message` or None\n" "\n" }, { "assign", (PyCFunction)Consumer_assign, METH_O, ".. py:function:: assign(partitions)\n" "\n" " Set consumer partition assignment to the provided list of " ":py:class:`TopicPartition` and starts consuming.\n" "\n" " :param list(TopicPartition) partitions: List of topic+partitions and optionally initial offsets to start consuming.\n" "\n" }, { "unassign", (PyCFunction)Consumer_unassign, METH_NOARGS, " Removes the current partition assignment and stops consuming.\n" "\n" }, { "commit", (PyCFunction)Consumer_commit, METH_VARARGS|METH_KEYWORDS, ".. py:function:: commit([message=None], [offsets=None], [async=True])\n" "\n" " Commit a message or a list of offsets.\n" "\n" " ``message`` and ``offsets`` are mutually exclusive, if neither is set " "the current partition assignment's offsets are used instead.\n" "\n" " :param confluent_kafka.Message message: Commit message's offset+1.\n" " :param list(TopicPartition) offsets: List of topic+partitions+offsets to commit.\n" " :param bool async: Asynchronous commit, return immediately.\n" " :rtype: None\n" " :raises: KafkaException\n" "\n" }, { "committed", (PyCFunction)Consumer_committed, METH_VARARGS|METH_KEYWORDS, ".. py:function:: committed(partitions, [timeout=None])\n" "\n" " Retrieve committed offsets for the list of partitions.\n" "\n" " :param list(TopicPartition) partitions: List of topic+partitions " "to query for stored offsets.\n" " :param float timeout: Request timeout\n" " :returns: List of topic+partitions with offset and possibly error set.\n" " :rtype: list(TopicPartition)\n" " :raises: KafkaException\n" "\n" }, { "position", (PyCFunction)Consumer_position, METH_VARARGS|METH_KEYWORDS, ".. py:function:: position(partitions, [timeout=None])\n" "\n" " Retrieve current positions (offsets) for the list of partitions.\n" "\n" " :param list(TopicPartition) partitions: List of topic+partitions " "to return current offsets for. The current offset is the offset of the " "last consumed message + 1.\n" " :returns: List of topic+partitions with offset and possibly error set.\n" " :rtype: list(TopicPartition)\n" " :raises: KafkaException\n" "\n" }, { "close", (PyCFunction)Consumer_close, METH_NOARGS, "\n" " Close down and terminate the Kafka Consumer.\n" "\n" " Actions performed:\n" "\n" " - Stops consuming\n" " - Commits offsets\n" " - Leave consumer group\n" "\n" " .. note: Registered callbacks may be called from this method, " "see :py:func::`poll()` for more info.\n" "\n" " :rtype: None\n" "\n" }, { NULL } }; static PyObject *Consumer_new (PyTypeObject *type, PyObject *args, PyObject *kwargs); PyTypeObject ConsumerType = { PyVarObject_HEAD_INIT(NULL, 0) "cimpl.Consumer", /*tp_name*/ sizeof(Handle), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)Consumer_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ "High-level Kafka Consumer\n" "\n" ".. py:function:: Consumer(**kwargs)\n" "\n" " Create new Consumer instance using provided configuration dict.\n" "\n" " Special configuration properties:\n" " ``on_commit``: Optional callback will be called when a commit " "request has succeeded or failed.\n" "\n" "\n" ".. py:function:: on_commit(err, partitions)\n" "\n" " :param Consumer consumer: Consumer instance.\n" " :param KafkaError err: Commit error object, or None on success.\n" " :param list(TopicPartition) partitions: List of partitions with " "their committed offsets or per-partition errors.\n" "\n" "\n", /*tp_doc*/ (traverseproc)Consumer_traverse, /* tp_traverse */ (inquiry)Consumer_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ Consumer_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ Consumer_new /* tp_new */ }; static void Consumer_rebalance_cb (rd_kafka_t *rk, rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t *c_parts, void *opaque) { Handle *self = opaque; CallState *cs; cs = CallState_get(self); self->u.Consumer.rebalance_assigned = 0; if ((err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS && self->u.Consumer.on_assign) || (err == RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS && self->u.Consumer.on_revoke)) { PyObject *parts; PyObject *args, *result; /* Construct list of TopicPartition based on 'c_parts' */ parts = c_parts_to_py(c_parts); args = Py_BuildValue("(OO)", self, parts); Py_DECREF(parts); if (!args) { cfl_PyErr_Format(RD_KAFKA_RESP_ERR__FAIL, "Unable to build callback args"); CallState_crash(cs); CallState_resume(cs); return; } result = PyObject_CallObject( err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS ? self->u.Consumer.on_assign : self->u.Consumer.on_revoke, args); Py_DECREF(args); if (result) Py_DECREF(result); else { CallState_crash(cs); rd_kafka_yield(rk); } } /* Fallback: librdkafka needs the rebalance_cb to call assign() * to synchronize state, if the user did not do this from callback, * or there was no callback, or the callback failed, then we perform * that assign() call here instead. */ if (!self->u.Consumer.rebalance_assigned) { if (err == RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) rd_kafka_assign(rk, c_parts); else rd_kafka_assign(rk, NULL); } CallState_resume(cs); } static void Consumer_offset_commit_cb (rd_kafka_t *rk, rd_kafka_resp_err_t err, rd_kafka_topic_partition_list_t *c_parts, void *opaque) { Handle *self = opaque; PyObject *parts, *k_err, *args, *result; CallState *cs; if (!self->u.Consumer.on_commit) return; cs = CallState_get(self); /* Insantiate error object */ k_err = KafkaError_new_or_None(err, NULL); /* Construct list of TopicPartition based on 'c_parts' */ if (c_parts) parts = c_parts_to_py(c_parts); else parts = PyList_New(0); args = Py_BuildValue("(OO)", k_err, parts); Py_DECREF(k_err); Py_DECREF(parts); if (!args) { cfl_PyErr_Format(RD_KAFKA_RESP_ERR__FAIL, "Unable to build callback args"); CallState_crash(cs); CallState_resume(cs); return; } result = PyObject_CallObject(self->u.Consumer.on_commit, args); Py_DECREF(args); if (result) Py_DECREF(result); else { CallState_crash(cs); rd_kafka_yield(rk); } CallState_resume(cs); } static PyObject *Consumer_new (PyTypeObject *type, PyObject *args, PyObject *kwargs) { Handle *self; char errstr[256]; rd_kafka_conf_t *conf; self = (Handle *)ConsumerType.tp_alloc(&ConsumerType, 0); if (!self) return NULL; if (!(conf = common_conf_setup(RD_KAFKA_CONSUMER, self, args, kwargs))) { Py_DECREF(self); return NULL; } rd_kafka_conf_set_rebalance_cb(conf, Consumer_rebalance_cb); rd_kafka_conf_set_offset_commit_cb(conf, Consumer_offset_commit_cb); self->rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof(errstr)); if (!self->rk) { cfl_PyErr_Format(rd_kafka_last_error(), "Failed to create consumer: %s", errstr); Py_DECREF(self); return NULL; } rd_kafka_poll_set_consumer(self->rk); return (PyObject *)self; } python-confluent-kafka-0.9.2/confluent_kafka/src/Producer.c000066400000000000000000000320661303570112400237640ustar00rootroot00000000000000/** * Copyright 2016 Confluent Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "confluent_kafka.h" /** * @brief KNOWN ISSUES * * - Partitioners will cause a dead-lock with librdkafka, because: * GIL + topic lock in topic_new is different lock order than * topic lock in msg_partitioner + GIL. * This needs to be sorted out in librdkafka, preferably making the * partitioner run without any locks taken. * Until this is fixed the partitioner is ignored and librdkafka's * default will be used. * */ /**************************************************************************** * * * Producer * * * * ****************************************************************************/ /** * Per-message state. */ struct Producer_msgstate { Handle *self; PyObject *dr_cb; PyObject *partitioner_cb; }; /** * Create a new per-message state. * Returns NULL if neither dr_cb or partitioner_cb is set. */ static __inline struct Producer_msgstate * Producer_msgstate_new (Handle *self, PyObject *dr_cb, PyObject *partitioner_cb) { struct Producer_msgstate *msgstate; if (!dr_cb && !partitioner_cb) return NULL; msgstate = calloc(1, sizeof(*msgstate)); msgstate->self = self; if (dr_cb) { msgstate->dr_cb = dr_cb; Py_INCREF(dr_cb); } if (partitioner_cb) { msgstate->partitioner_cb = partitioner_cb; Py_INCREF(partitioner_cb); } return msgstate; } static __inline void Producer_msgstate_destroy (struct Producer_msgstate *msgstate) { if (msgstate->dr_cb) Py_DECREF(msgstate->dr_cb); if (msgstate->partitioner_cb) Py_DECREF(msgstate->partitioner_cb); free(msgstate); } static int Producer_clear (Handle *self) { if (self->u.Producer.default_dr_cb) { Py_DECREF(self->u.Producer.default_dr_cb); self->u.Producer.default_dr_cb = NULL; } if (self->u.Producer.partitioner_cb) { Py_DECREF(self->u.Producer.partitioner_cb); self->u.Producer.partitioner_cb = NULL; } Handle_clear(self); return 0; } static void Producer_dealloc (Handle *self) { PyObject_GC_UnTrack(self); Producer_clear(self); if (self->rk) rd_kafka_destroy(self->rk); Py_TYPE(self)->tp_free((PyObject *)self); } static int Producer_traverse (Handle *self, visitproc visit, void *arg) { if (self->u.Producer.default_dr_cb) Py_VISIT(self->u.Producer.default_dr_cb); if (self->u.Producer.partitioner_cb) Py_VISIT(self->u.Producer.partitioner_cb); Handle_traverse(self, visit, arg); return 0; } static void dr_msg_cb (rd_kafka_t *rk, const rd_kafka_message_t *rkm, void *opaque) { struct Producer_msgstate *msgstate = rkm->_private; Handle *self = opaque; CallState *cs; PyObject *args; PyObject *result; PyObject *msgobj; if (!msgstate) return; cs = CallState_get(self); if (!msgstate->dr_cb) { /* No callback defined */ goto done; } msgobj = Message_new0(rkm); args = Py_BuildValue("(OO)", Message_error((Message *)msgobj, NULL), msgobj); Py_DECREF(msgobj); if (!args) { cfl_PyErr_Format(RD_KAFKA_RESP_ERR__FAIL, "Unable to build callback args"); CallState_crash(cs); goto done; } result = PyObject_CallObject(msgstate->dr_cb, args); Py_DECREF(args); if (result) Py_DECREF(result); else { CallState_crash(cs); rd_kafka_yield(rk); } done: Producer_msgstate_destroy(msgstate); CallState_resume(cs); } /** * FIXME: The partitioner is currently broken due to threading/GIL issues. */ int32_t Producer_partitioner_cb (const rd_kafka_topic_t *rkt, const void *keydata, size_t keylen, int32_t partition_cnt, void *rkt_opaque, void *msg_opaque) { Handle *self = rkt_opaque; struct Producer_msgstate *msgstate = msg_opaque; PyGILState_STATE gstate; PyObject *result; PyObject *args; int32_t r = RD_KAFKA_PARTITION_UA; if (!msgstate) { /* Fall back on default C partitioner if neither a per-msg * partitioner nor a default Python partitioner is available */ return self->u.Producer.c_partitioner_cb(rkt, keydata, keylen, partition_cnt, rkt_opaque, msg_opaque); } gstate = PyGILState_Ensure(); if (!msgstate->partitioner_cb) { /* Fall back on default C partitioner if neither a per-msg * partitioner nor a default Python partitioner is available */ r = msgstate->self->u.Producer.c_partitioner_cb(rkt, keydata, keylen, partition_cnt, rkt_opaque, msg_opaque); goto done; } args = Py_BuildValue("(s#l)", (const char *)keydata, (int)keylen, (long)partition_cnt); if (!args) { cfl_PyErr_Format(RD_KAFKA_RESP_ERR__FAIL, "Unable to build callback args"); goto done; } result = PyObject_CallObject(msgstate->partitioner_cb, args); Py_DECREF(args); if (result) { r = (int32_t)PyLong_AsLong(result); if (PyErr_Occurred()) printf("FIXME: partition_cb returned wrong type " "(expected long), how to propagate?\n"); Py_DECREF(result); } else { printf("FIXME: partitioner_cb crashed, how to propagate?\n"); } done: PyGILState_Release(gstate); return r; } static PyObject *Producer_produce (Handle *self, PyObject *args, PyObject *kwargs) { const char *topic, *value = NULL, *key = NULL; int value_len = 0, key_len = 0; int partition = RD_KAFKA_PARTITION_UA; PyObject *dr_cb = NULL, *dr_cb2 = NULL, *partitioner_cb = NULL; rd_kafka_topic_t *rkt; struct Producer_msgstate *msgstate; static char *kws[] = { "topic", "value", "key", "partition", "callback", "on_delivery", /* Alias */ "partitioner", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|z#z#iOOO", kws, &topic, &value, &value_len, &key, &key_len, &partition, &dr_cb, &dr_cb2, &partitioner_cb)) return NULL; if (dr_cb2 && !dr_cb) /* Alias */ dr_cb = dr_cb2; if (!(rkt = rd_kafka_topic_new(self->rk, topic, NULL))) { cfl_PyErr_Format(rd_kafka_last_error(), "Unable to create topic object: %s", rd_kafka_err2str(rd_kafka_last_error())); return NULL; } if (!dr_cb || dr_cb == Py_None) dr_cb = self->u.Producer.default_dr_cb; if (!partitioner_cb || partitioner_cb == Py_None) partitioner_cb = self->u.Producer.partitioner_cb; /* Create msgstate if necessary, may return NULL if no callbacks * are wanted. */ msgstate = Producer_msgstate_new(self, dr_cb, partitioner_cb); /* Produce message */ if (rd_kafka_produce(rkt, partition, RD_KAFKA_MSG_F_COPY, (void *)value, value_len, (void *)key, key_len, msgstate) == -1) { rd_kafka_resp_err_t err = rd_kafka_last_error(); if (msgstate) Producer_msgstate_destroy(msgstate); if (err == RD_KAFKA_RESP_ERR__QUEUE_FULL) PyErr_Format(PyExc_BufferError, "%s", rd_kafka_err2str(err)); else cfl_PyErr_Format(err, "Unable to produce message: %s", rd_kafka_err2str(err)); return NULL; } rd_kafka_topic_destroy(rkt); Py_RETURN_NONE; } /** * @brief Call rd_kafka_poll() and keep track of crashing callbacks. * @returns -1 if callback crashed (or poll() failed), else the number * of events served. */ static int Producer_poll0 (Handle *self, int tmout) { int r; CallState cs; CallState_begin(self, &cs); r = rd_kafka_poll(self->rk, tmout); if (!CallState_end(self, &cs)) { return -1; } return r; } static PyObject *Producer_poll (Handle *self, PyObject *args, PyObject *kwargs) { double tmout; int r; static char *kws[] = { "timeout", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "d", kws, &tmout)) return NULL; r = Producer_poll0(self, (int)(tmout * 1000)); if (r == -1) return NULL; return PyLong_FromLong(r); } static PyObject *Producer_flush (Handle *self, PyObject *ignore) { while (rd_kafka_outq_len(self->rk) > 0) { if (Producer_poll0(self, 500) == -1) return NULL; } Py_RETURN_NONE; } static PyMethodDef Producer_methods[] = { { "produce", (PyCFunction)Producer_produce, METH_VARARGS|METH_KEYWORDS, ".. py:function:: produce(topic, [value], [key], [partition], [callback])\n" "\n" " Produce message to topic.\n" " This is an asynchronous operation, an application may use the " "``callback`` (alias ``on_delivery``) argument to pass a function " "(or lambda) that will be called from :py:func:`poll()` when the " "message has been successfully delivered or permanently fails delivery.\n" "\n" " :param str topic: Topic to produce message to\n" " :param str|bytes value: Message payload\n" " :param str|bytes key: Message key\n" " :param int partition: Partition to produce to, elses uses the " "configured partitioner.\n" " :param func on_delivery(err,msg): Delivery report callback to call " "(from :py:func:`poll()` or :py:func:`flush()`) on successful or " "failed delivery\n" "\n" " :rtype: None\n" " :raises BufferError: if the internal producer message queue is " "full (``queue.buffering.max.messages`` exceeded)\n" " :raises KafkaException: for other errors, see exception code\n" "\n" }, { "poll", (PyCFunction)Producer_poll, METH_VARARGS|METH_KEYWORDS, ".. py:function:: poll([timeout])\n" "\n" " Polls the producer for events and calls the corresponding " "callbacks (if registered).\n" "\n" " Callbacks:\n" "\n" " - ``on_delivery`` callbacks from :py:func:`produce()`\n" " - ...\n" "\n" " :param float timeout: Maximum time to block waiting for events.\n" " :returns: Number of events processed (callbacks served)\n" " :rtype: int\n" "\n" }, { "flush", (PyCFunction)Producer_flush, METH_NOARGS, " Wait for all messages in the Producer queue to be delivered.\n" " This is a convenience method that calls :py:func:`poll()` until " ":py:func:`len()` is zero.\n" "\n" ".. note:: See :py:func:`poll()` for a description on what " "callbacks may be triggered.\n" "\n" }, { NULL } }; static Py_ssize_t Producer__len__ (Handle *self) { return rd_kafka_outq_len(self->rk); } static PySequenceMethods Producer_seq_methods = { (lenfunc)Producer__len__ /* sq_length */ }; static PyObject *Producer_new (PyTypeObject *type, PyObject *args, PyObject *kwargs); PyTypeObject ProducerType = { PyVarObject_HEAD_INIT(NULL, 0) "cimpl.Producer", /*tp_name*/ sizeof(Handle), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)Producer_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ &Producer_seq_methods, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ "Asynchronous Kafka Producer\n" "\n" ".. py:function:: Producer(**kwargs)\n" "\n" " Create new Producer instance using provided configuration dict.\n" "\n" "\n" ".. py:function:: len()\n" "\n" " :returns: Number of messages and Kafka protocol requests waiting to be delivered to broker.\n" " :rtype: int\n" "\n", /*tp_doc*/ (traverseproc)Producer_traverse, /* tp_traverse */ (inquiry)Producer_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ Producer_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ Producer_new /* tp_new */ }; static PyObject *Producer_new (PyTypeObject *type, PyObject *args, PyObject *kwargs) { Handle *self; char errstr[256]; rd_kafka_conf_t *conf; self = (Handle *)ProducerType.tp_alloc(&ProducerType, 0); if (!self) return NULL; if (!(conf = common_conf_setup(RD_KAFKA_PRODUCER, self, args, kwargs))) { Py_DECREF(self); return NULL; } rd_kafka_conf_set_dr_msg_cb(conf, dr_msg_cb); self->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof(errstr)); if (!self->rk) { cfl_PyErr_Format(rd_kafka_last_error(), "Failed to create producer: %s", errstr); Py_DECREF(self); return NULL; } return (PyObject *)self; } python-confluent-kafka-0.9.2/confluent_kafka/src/confluent_kafka.c000066400000000000000000001050151303570112400253260ustar00rootroot00000000000000/** * Copyright 2016 Confluent Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "confluent_kafka.h" #include /** * @brief KNOWN ISSUES * * - Partitioners will cause a dead-lock with librdkafka, because: * GIL + topic lock in topic_new is different lock order than * topic lock in msg_partitioner + GIL. * This needs to be sorted out in librdkafka, preferably making the * partitioner run without any locks taken. * Until this is fixed the partitioner is ignored and librdkafka's * default will be used. * - TopicPartition.offset should probably be None for the INVALID offset * rather than exposing the special value -1001. * - KafkaError type .tp_doc allocation is lost on exit. * */ PyObject *KafkaException; /**************************************************************************** * * * KafkaError * * * FIXME: Pre-create simple instances for each error code, only instantiate * a new object if a rich error string is provided. * ****************************************************************************/ typedef struct { PyObject_HEAD rd_kafka_resp_err_t code; /* Error code */ char *str; /* Human readable representation of error, if one * was provided by librdkafka. * Else falls back on err2str(). */ } KafkaError; static PyObject *KafkaError_code (KafkaError *self, PyObject *ignore) { return PyLong_FromLong(self->code); } static PyObject *KafkaError_str (KafkaError *self, PyObject *ignore) { if (self->str) return cfl_PyUnistr(_FromString(self->str)); else return cfl_PyUnistr(_FromString(rd_kafka_err2str(self->code))); } static PyObject *KafkaError_name (KafkaError *self, PyObject *ignore) { /* FIXME: Pre-create name objects */ return cfl_PyUnistr(_FromString(rd_kafka_err2name(self->code))); } static PyMethodDef KafkaError_methods[] = { { "code", (PyCFunction)KafkaError_code, METH_NOARGS, " Returns the error/event code for comparison to" "KafkaError..\n" "\n" " :returns: error/event code\n" " :rtype: int\n" "\n" }, { "str", (PyCFunction)KafkaError_str, METH_NOARGS, " Returns the human-readable error/event string.\n" "\n" " :returns: error/event message string\n" " :rtype: str\n" "\n" }, { "name", (PyCFunction)KafkaError_name, METH_NOARGS, " Returns the enum name for error/event.\n" "\n" " :returns: error/event enum name string\n" " :rtype: str\n" "\n" }, { NULL } }; static void KafkaError_dealloc (KafkaError *self) { if (self->str) free(self->str); Py_TYPE(self)->tp_free((PyObject *)self); } static PyObject *KafkaError_str0 (KafkaError *self) { return cfl_PyUnistr(_FromFormat("KafkaError{code=%s,val=%d,str=\"%s\"}", rd_kafka_err2name(self->code), self->code, self->str ? self->str : rd_kafka_err2str(self->code))); } static long KafkaError_hash (KafkaError *self) { return self->code; } static PyTypeObject KafkaErrorType; static PyObject* KafkaError_richcompare (KafkaError *self, PyObject *o2, int op) { int code2; int r; PyObject *result; if (Py_TYPE(o2) == &KafkaErrorType) code2 = ((KafkaError *)o2)->code; else code2 = (int)PyLong_AsLong(o2); switch (op) { case Py_LT: r = self->code < code2; break; case Py_LE: r = self->code <= code2; break; case Py_EQ: r = self->code == code2; break; case Py_NE: r = self->code != code2; break; case Py_GT: r = self->code > code2; break; case Py_GE: r = self->code >= code2; break; default: r = 0; break; } result = r ? Py_True : Py_False; Py_INCREF(result); return result; } static PyTypeObject KafkaErrorType = { PyVarObject_HEAD_INIT(NULL, 0) "cimpl.KafkaError", /*tp_name*/ sizeof(KafkaError), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)KafkaError_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ (reprfunc)KafkaError_str0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ (hashfunc)KafkaError_hash, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT, /*tp_flags*/ "Kafka error and event object\n" "\n" " The KafkaError class serves multiple purposes:\n" "\n" " - Propagation of errors\n" " - Propagation of events\n" " - Exceptions\n" "\n" " This class is not user-instantiable.\n" "\n", /*tp_doc*/ 0, /* tp_traverse */ 0, /* tp_clear */ (richcmpfunc)KafkaError_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ KafkaError_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0 /* tp_alloc */ }; /** * @brief Initialize a KafkaError object. */ static void KafkaError_init (KafkaError *self, rd_kafka_resp_err_t code, const char *str) { self->code = code; if (str) self->str = strdup(str); else self->str = NULL; } /** * @brief Internal factory to create KafkaError object. */ PyObject *KafkaError_new0 (rd_kafka_resp_err_t err, const char *fmt, ...) { KafkaError *self; va_list ap; char buf[512]; self = (KafkaError *)KafkaErrorType. tp_alloc(&KafkaErrorType, 0); if (!self) return NULL; if (fmt) { va_start(ap, fmt); vsnprintf(buf, sizeof(buf), fmt, ap); va_end(ap); } KafkaError_init(self, err, fmt ? buf : rd_kafka_err2str(err)); return (PyObject *)self; } /** * @brief Internal factory to create KafkaError object. * @returns a new KafkaError object if \p err != 0, else a None object. */ PyObject *KafkaError_new_or_None (rd_kafka_resp_err_t err, const char *str) { if (!err) Py_RETURN_NONE; return KafkaError_new0(err, "%s", str); } /**************************************************************************** * * * Message * * * * ****************************************************************************/ PyObject *Message_error (Message *self, PyObject *ignore) { if (self->error) { Py_INCREF(self->error); return self->error; } else Py_RETURN_NONE; } static PyObject *Message_value (Message *self, PyObject *ignore) { if (self->value) { Py_INCREF(self->value); return self->value; } else Py_RETURN_NONE; } static PyObject *Message_key (Message *self, PyObject *ignore) { if (self->key) { Py_INCREF(self->key); return self->key; } else Py_RETURN_NONE; } static PyObject *Message_topic (Message *self, PyObject *ignore) { if (self->topic) { Py_INCREF(self->topic); return self->topic; } else Py_RETURN_NONE; } static PyObject *Message_partition (Message *self, PyObject *ignore) { if (self->partition != RD_KAFKA_PARTITION_UA) return PyLong_FromLong(self->partition); else Py_RETURN_NONE; } static PyObject *Message_offset (Message *self, PyObject *ignore) { if (self->offset >= 0) return PyLong_FromLongLong(self->offset); else Py_RETURN_NONE; } static PyMethodDef Message_methods[] = { { "error", (PyCFunction)Message_error, METH_NOARGS, " The message object is also used to propagate errors and events, " "an application must check error() to determine if the Message " "is a proper message (error() returns None) or an error or event " "(error() returns a KafkaError object)\n" "\n" " :rtype: None or :py:class:`KafkaError`\n" "\n" }, { "value", (PyCFunction)Message_value, METH_NOARGS, " :returns: message value (payload) or None if not available.\n" " :rtype: str|bytes or None\n" "\n" }, { "key", (PyCFunction)Message_key, METH_NOARGS, " :returns: message key or None if not available.\n" " :rtype: str|bytes or None\n" "\n" }, { "topic", (PyCFunction)Message_topic, METH_NOARGS, " :returns: topic name or None if not available.\n" " :rtype: str or None\n" "\n" }, { "partition", (PyCFunction)Message_partition, METH_NOARGS, " :returns: partition number or None if not available.\n" " :rtype: int or None\n" "\n" }, { "offset", (PyCFunction)Message_offset, METH_NOARGS, " :returns: message offset or None if not available.\n" " :rtype: int or None\n" "\n" }, { NULL } }; static int Message_clear (Message *self) { if (self->topic) { Py_DECREF(self->topic); self->topic = NULL; } if (self->value) { Py_DECREF(self->value); self->value = NULL; } if (self->key) { Py_DECREF(self->key); self->key = NULL; } if (self->error) { Py_DECREF(self->error); self->error = NULL; } return 0; } static void Message_dealloc (Message *self) { Message_clear(self); PyObject_GC_UnTrack(self); Py_TYPE(self)->tp_free((PyObject *)self); } static int Message_traverse (Message *self, visitproc visit, void *arg) { if (self->topic) Py_VISIT(self->topic); if (self->value) Py_VISIT(self->value); if (self->key) Py_VISIT(self->key); if (self->error) Py_VISIT(self->error); return 0; } static Py_ssize_t Message__len__ (Message *self) { return self->value ? PyObject_Length(self->value) : 0; } static PySequenceMethods Message_seq_methods = { (lenfunc)Message__len__ /* sq_length */ }; PyTypeObject MessageType = { PyVarObject_HEAD_INIT(NULL, 0) "cimpl.Message", /*tp_name*/ sizeof(Message), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)Message_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ &Message_seq_methods, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ "The Message object represents either a single consumed or " "produced message, or an event (:py:func:`error()` is not None).\n" "\n" "An application must check with :py:func:`error()` to see if the " "object is a proper message (error() returns None) or an " "error/event.\n" "\n" "This class is not user-instantiable.\n" "\n" ".. py:function:: len()\n" "\n" " :returns: Message value (payload) size in bytes\n" " :rtype: int\n" "\n", /*tp_doc*/ (traverseproc)Message_traverse, /* tp_traverse */ (inquiry)Message_clear, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ Message_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0 /* tp_alloc */ }; /** * @brief Internal factory to create Message object from message_t */ PyObject *Message_new0 (const rd_kafka_message_t *rkm) { Message *self; self = (Message *)MessageType.tp_alloc(&MessageType, 0); if (!self) return NULL; self->error = KafkaError_new_or_None(rkm->err, rkm->err ? rd_kafka_message_errstr(rkm) : NULL); if (rkm->rkt) self->topic = cfl_PyUnistr( _FromString(rd_kafka_topic_name(rkm->rkt))); if (rkm->payload) self->value = cfl_PyBin(_FromStringAndSize(rkm->payload, rkm->len)); if (rkm->key) self->key = cfl_PyBin( _FromStringAndSize(rkm->key, rkm->key_len)); self->partition = rkm->partition; self->offset = rkm->offset; return (PyObject *)self; } /**************************************************************************** * * * TopicPartition * * * * ****************************************************************************/ typedef struct { PyObject_HEAD char *topic; int partition; int64_t offset; PyObject *error; } TopicPartition; static int TopicPartition_clear (TopicPartition *self) { if (self->topic) { free(self->topic); self->topic = NULL; } if (self->error) { Py_DECREF(self->error); self->error = NULL; } return 0; } static void TopicPartition_dealloc (TopicPartition *self) { PyObject_GC_UnTrack(self); TopicPartition_clear(self); Py_TYPE(self)->tp_free((PyObject *)self); } static PyObject *TopicPartition_new0 (const char *topic, int partition, long long offset, rd_kafka_resp_err_t err); static PyObject *TopicPartition_new (PyTypeObject *type, PyObject *args, PyObject *kwargs) { const char *topic; int partition = RD_KAFKA_PARTITION_UA; long long offset = RD_KAFKA_OFFSET_INVALID; static char *kws[] = { "topic", "partition", "offset", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|iL", kws, &topic, &partition, &offset)) return NULL; return TopicPartition_new0(topic, partition, offset, 0); } static int TopicPartition_traverse (TopicPartition *self, visitproc visit, void *arg) { if (self->error) Py_VISIT(self->error); return 0; } static PyMemberDef TopicPartition_members[] = { { "topic", T_STRING, offsetof(TopicPartition, topic), READONLY, "Topic name" }, { "partition", T_INT, offsetof(TopicPartition, partition), 0, "Partition number" }, { "offset", T_LONGLONG, offsetof(TopicPartition, offset), 0, "Offset" }, /* FIXME: Possibly use None for INVALID offset (-1001) */ { "error", T_OBJECT, offsetof(TopicPartition, error), READONLY, "Indicates an error (with :py:class:`KafkaError`) unless None." }, { NULL } }; static PyObject *TopicPartition_str0 (TopicPartition *self) { PyObject *errstr = self->error == Py_None ? NULL : cfl_PyObject_Unistr(self->error); PyObject *ret; ret = cfl_PyUnistr( _FromFormat("TopicPartition{topic=%s,partition=%"PRId32 ",offset=%"PRId64",error=%s}", self->topic, self->partition, self->offset, errstr ? cfl_PyUnistr_AsUTF8(errstr) : "None")); if (errstr) Py_DECREF(errstr); return ret; } static PyTypeObject TopicPartitionType; static PyObject * TopicPartition_richcompare (TopicPartition *self, PyObject *o2, int op) { TopicPartition *a = self, *b; int tr, pr; int r; PyObject *result; if (Py_TYPE(o2) != Py_TYPE(self)) { PyErr_SetNone(PyExc_NotImplementedError); return NULL; } b = (TopicPartition *)o2; tr = strcmp(a->topic, b->topic); pr = a->partition - b->partition; switch (op) { case Py_LT: r = tr < 0 || (tr == 0 && pr < 0); break; case Py_LE: r = tr < 0 || (tr == 0 && pr <= 0); break; case Py_EQ: r = (tr == 0 && pr == 0); break; case Py_NE: r = (tr != 0 || pr != 0); break; case Py_GT: r = tr > 0 || (tr == 0 && pr > 0); break; case Py_GE: r = tr > 0 || (tr == 0 && pr >= 0); break; default: r = 0; break; } result = r ? Py_True : Py_False; Py_INCREF(result); return result; } static long TopicPartition_hash (TopicPartition *self) { PyObject *topic = cfl_PyUnistr(_FromString(self->topic)); long r = PyObject_Hash(topic) ^ self->partition; Py_DECREF(topic); return r; } static PyTypeObject TopicPartitionType = { PyVarObject_HEAD_INIT(NULL, 0) "cimpl.TopicPartition", /*tp_name*/ sizeof(TopicPartition), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor)TopicPartition_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ (reprfunc)TopicPartition_str0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ (hashfunc)TopicPartition_hash, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ "TopicPartition is a generic type to hold a single partition and " "various information about it.\n" "\n" "It is typically used to provide a list of topics or partitions for " "various operations, such as :py:func:`Consumer.assign()`.\n" "\n" ".. py:function:: TopicPartition(topic, [partition], [offset])\n" "\n" " Instantiate a TopicPartition object.\n" "\n" " :param string topic: Topic name\n" " :param int partition: Partition id\n" " :param int offset: Initial partition offset\n" " :rtype: TopicPartition\n" "\n" "\n", /*tp_doc*/ (traverseproc)TopicPartition_traverse, /* tp_traverse */ (inquiry)TopicPartition_clear, /* tp_clear */ (richcmpfunc)TopicPartition_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ 0, /* tp_methods */ TopicPartition_members,/* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ TopicPartition_new /* tp_new */ }; /** * @brief Internal factory to create a TopicPartition object. */ static PyObject *TopicPartition_new0 (const char *topic, int partition, long long offset, rd_kafka_resp_err_t err) { TopicPartition *self; self = (TopicPartition *)TopicPartitionType.tp_alloc( &TopicPartitionType, 0); if (!self) return NULL; self->topic = strdup(topic); self->partition = partition; self->offset = offset; self->error = KafkaError_new_or_None(err, NULL); return (PyObject *)self; } /** * @brief Convert C rd_kafka_topic_partition_list_t to Python list(TopicPartition). * * @returns The new Python list object. */ PyObject *c_parts_to_py (const rd_kafka_topic_partition_list_t *c_parts) { PyObject *parts; size_t i; parts = PyList_New(c_parts->cnt); for (i = 0 ; i < c_parts->cnt ; i++) { const rd_kafka_topic_partition_t *rktpar = &c_parts->elems[i]; PyList_SET_ITEM(parts, i, TopicPartition_new0( rktpar->topic, rktpar->partition, rktpar->offset, rktpar->err)); } return parts; } /** * @brief Convert Python list(TopicPartition) to C rd_kafka_topic_partition_list_t. * * @returns The new C list on success or NULL on error. */ rd_kafka_topic_partition_list_t *py_to_c_parts (PyObject *plist) { rd_kafka_topic_partition_list_t *c_parts; size_t i; if (!PyList_Check(plist)) { PyErr_SetString(PyExc_TypeError, "requires list of TopicPartition"); return NULL; } c_parts = rd_kafka_topic_partition_list_new((int)PyList_Size(plist)); for (i = 0 ; i < PyList_Size(plist) ; i++) { TopicPartition *tp = (TopicPartition *) PyList_GetItem(plist, i); if (PyObject_Type((PyObject *)tp) != (PyObject *)&TopicPartitionType) { PyErr_Format(PyExc_TypeError, "expected %s", TopicPartitionType.tp_name); rd_kafka_topic_partition_list_destroy(c_parts); return NULL; } rd_kafka_topic_partition_list_add(c_parts, tp->topic, tp->partition)->offset = tp->offset; } return c_parts; } /**************************************************************************** * * * Common callbacks * * * * ****************************************************************************/ static void error_cb (rd_kafka_t *rk, int err, const char *reason, void *opaque) { Handle *h = opaque; PyObject *eo, *result; CallState *cs; cs = CallState_get(h); if (!h->error_cb) { /* No callback defined */ goto done; } eo = KafkaError_new0(err, "%s", reason); result = PyObject_CallFunctionObjArgs(h->error_cb, eo, NULL); Py_DECREF(eo); if (result) { Py_DECREF(result); } else { CallState_crash(cs); rd_kafka_yield(h->rk); } done: CallState_resume(cs); } /**************************************************************************** * * * Common helpers * * * * ****************************************************************************/ /** * Clear Python object references in Handle */ void Handle_clear (Handle *h) { if (h->error_cb) { Py_DECREF(h->error_cb); } PyThread_delete_key(h->tlskey); } /** * GC traversal for Python object references */ int Handle_traverse (Handle *h, visitproc visit, void *arg) { if (h->error_cb) Py_VISIT(h->error_cb); return 0; } /** * Populate topic conf from provided dict. * * Will raise an exception on error and return -1, or returns 0 on success. */ static int populate_topic_conf (rd_kafka_topic_conf_t *tconf, const char *what, PyObject *dict) { Py_ssize_t pos = 0; PyObject *ko, *vo; if (!PyDict_Check(dict)) { cfl_PyErr_Format(RD_KAFKA_RESP_ERR__INVALID_ARG, "%s: requires a dict", what); return -1; } while (PyDict_Next(dict, &pos, &ko, &vo)) { PyObject *ks; PyObject *vs; const char *k; const char *v; char errstr[256]; if (!(ks = cfl_PyObject_Unistr(ko))) { PyErr_SetString(PyExc_TypeError, "expected configuration property " "value as type unicode string"); return -1; } if (!(vs = cfl_PyObject_Unistr(vo))) { PyErr_SetString(PyExc_TypeError, "expected configuration property " "value as type unicode string"); Py_DECREF(ks); return -1; } k = cfl_PyUnistr_AsUTF8(ks); v = cfl_PyUnistr_AsUTF8(vs); if (rd_kafka_topic_conf_set(tconf, k, v, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { cfl_PyErr_Format(RD_KAFKA_RESP_ERR__INVALID_ARG, "%s: %s", what, errstr); Py_DECREF(ks); Py_DECREF(vs); return -1; } Py_DECREF(ks); Py_DECREF(vs); } return 0; } /** * @brief Set single special producer config value. * * @returns 1 if handled, 0 if unknown, or -1 on failure (exception raised). */ static int producer_conf_set_special (Handle *self, rd_kafka_conf_t *conf, rd_kafka_topic_conf_t *tconf, const char *name, PyObject *valobj) { PyObject *vs; const char *val; if (!strcasecmp(name, "on_delivery")) { if (!PyCallable_Check(valobj)) { cfl_PyErr_Format( RD_KAFKA_RESP_ERR__INVALID_ARG, "%s requires a callable " "object", name); return -1; } self->u.Producer.default_dr_cb = valobj; Py_INCREF(self->u.Producer.default_dr_cb); return 1; } else if (!strcasecmp(name, "partitioner") || !strcasecmp(name, "partitioner_callback")) { if ((vs = cfl_PyObject_Unistr(valobj))) { /* Use built-in C partitioners, * based on their name. */ val = cfl_PyUnistr_AsUTF8(vs); if (!strcmp(val, "random")) rd_kafka_topic_conf_set_partitioner_cb( tconf, rd_kafka_msg_partitioner_random); else if (!strcmp(val, "consistent")) rd_kafka_topic_conf_set_partitioner_cb( tconf, rd_kafka_msg_partitioner_consistent); else if (!strcmp(val, "consistent_random")) rd_kafka_topic_conf_set_partitioner_cb( tconf, rd_kafka_msg_partitioner_consistent_random); else { cfl_PyErr_Format( RD_KAFKA_RESP_ERR__INVALID_ARG, "unknown builtin partitioner: %s " "(available: random, consistent, consistent_random)", val); Py_DECREF(vs); return -1; } Py_DECREF(vs); } else { /* Custom partitioner (Python callback) */ if (!PyCallable_Check(valobj)) { cfl_PyErr_Format( RD_KAFKA_RESP_ERR__INVALID_ARG, "%s requires a callable " "object", name); return -1; } /* FIXME: Error out until GIL+rdkafka lock-ordering is fixed. */ if (1) { cfl_PyErr_Format( RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED, "custom partitioner support not yet implemented"); return -1; } if (self->u.Producer.partitioner_cb) Py_DECREF(self->u.Producer.partitioner_cb); self->u.Producer.partitioner_cb = valobj; Py_INCREF(self->u.Producer.partitioner_cb); /* Use trampoline to call Python code. */ rd_kafka_topic_conf_set_partitioner_cb(tconf, Producer_partitioner_cb); } return 1; } return 0; /* Not handled */ } /** * @brief Set single special consumer config value. * * @returns 1 if handled, 0 if unknown, or -1 on failure (exception raised). */ static int consumer_conf_set_special (Handle *self, rd_kafka_conf_t *conf, rd_kafka_topic_conf_t *tconf, const char *name, PyObject *valobj) { if (!strcasecmp(name, "on_commit")) { if (!PyCallable_Check(valobj)) { cfl_PyErr_Format( RD_KAFKA_RESP_ERR__INVALID_ARG, "%s requires a callable " "object", name); return -1; } self->u.Consumer.on_commit = valobj; Py_INCREF(self->u.Consumer.on_commit); return 1; } return 0; } /** * Common config setup for Kafka client handles. * * Returns a conf object on success or NULL on failure in which case * an exception has been raised. */ rd_kafka_conf_t *common_conf_setup (rd_kafka_type_t ktype, Handle *h, PyObject *args, PyObject *kwargs) { rd_kafka_conf_t *conf; rd_kafka_topic_conf_t *tconf; Py_ssize_t pos = 0; PyObject *ko, *vo; int32_t (*partitioner_cb) (const rd_kafka_topic_t *, const void *, size_t, int32_t, void *, void *) = partitioner_cb; if (!kwargs) { /* If no kwargs, fall back on single dict arg, if any. */ if (!args || !PyTuple_Check(args) || PyTuple_Size(args) < 1 || !PyDict_Check((kwargs = PyTuple_GetItem(args, 0)))) { PyErr_SetString(PyExc_TypeError, "expected configuration dict"); return NULL; } } conf = rd_kafka_conf_new(); tconf = rd_kafka_topic_conf_new(); /* Convert kwargs dict to config key-value pairs. */ while (PyDict_Next(kwargs, &pos, &ko, &vo)) { PyObject *ks; PyObject *vs = NULL; const char *k; const char *v; char errstr[256]; int r; if (!(ks = cfl_PyObject_Unistr(ko))) { PyErr_SetString(PyExc_TypeError, "expected configuration property name " "as type unicode string"); rd_kafka_topic_conf_destroy(tconf); rd_kafka_conf_destroy(conf); return NULL; } k = cfl_PyUnistr_AsUTF8(ks); if (!strcmp(k, "default.topic.config")) { if (populate_topic_conf(tconf, k, vo) == -1) { Py_DECREF(ks); rd_kafka_topic_conf_destroy(tconf); rd_kafka_conf_destroy(conf); return NULL; } Py_DECREF(ks); continue; } else if (!strcmp(k, "error_cb")) { if (h->error_cb) { Py_DECREF(h->error_cb); h->error_cb = NULL; } if (vo != Py_None) { h->error_cb = vo; Py_INCREF(h->error_cb); } Py_DECREF(ks); continue; } /* Special handling for certain config keys. */ if (ktype == RD_KAFKA_PRODUCER) r = producer_conf_set_special(h, conf, tconf, k, vo); else r = consumer_conf_set_special(h, conf, tconf, k, vo); if (r == -1) { /* Error */ Py_DECREF(ks); rd_kafka_topic_conf_destroy(tconf); rd_kafka_conf_destroy(conf); return NULL; } else if (r == 1) { /* Handled */ continue; } /* * Pass configuration property through to librdkafka. */ if (!(vs = cfl_PyObject_Unistr(vo))) { PyErr_SetString(PyExc_TypeError, "expected configuration property " "value as type unicode string"); rd_kafka_topic_conf_destroy(tconf); rd_kafka_conf_destroy(conf); Py_DECREF(ks); return NULL; } v = cfl_PyUnistr_AsUTF8(vs); if (rd_kafka_conf_set(conf, k, v, errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { cfl_PyErr_Format(RD_KAFKA_RESP_ERR__INVALID_ARG, "%s", errstr); rd_kafka_topic_conf_destroy(tconf); rd_kafka_conf_destroy(conf); Py_DECREF(vs); Py_DECREF(ks); return NULL; } Py_DECREF(vs); Py_DECREF(ks); } if (h->error_cb) rd_kafka_conf_set_error_cb(conf, error_cb); rd_kafka_topic_conf_set_opaque(tconf, h); rd_kafka_conf_set_default_topic_conf(conf, tconf); rd_kafka_conf_set_opaque(conf, h); h->tlskey = PyThread_create_key(); return conf; } /** * @brief Initialiase a CallState and unlock the GIL prior to a * possibly blocking external call. */ void CallState_begin (Handle *h, CallState *cs) { cs->thread_state = PyEval_SaveThread(); cs->crashed = 0; PyThread_set_key_value(h->tlskey, cs); } /** * @brief Relock the GIL after external call is done. * @returns 0 if a Python signal was raised or a callback crashed, else 1. */ int CallState_end (Handle *h, CallState *cs) { PyThread_delete_key_value(h->tlskey); PyEval_RestoreThread(cs->thread_state); if (PyErr_CheckSignals() == -1 || cs->crashed) return 0; return 1; } /** * @brief Get the current thread's CallState and re-locks the GIL. */ CallState *CallState_get (Handle *h) { CallState *cs = PyThread_get_key_value(h->tlskey); assert(cs != NULL); PyEval_RestoreThread(cs->thread_state); cs->thread_state = NULL; return cs; } /** * @brief Un-locks the GIL to resume blocking external call. */ void CallState_resume (CallState *cs) { assert(cs->thread_state == NULL); cs->thread_state = PyEval_SaveThread(); } /** * @brief Indicate that call crashed. */ void CallState_crash (CallState *cs) { cs->crashed++; } /**************************************************************************** * * * Base * * * * ****************************************************************************/ static PyObject *libversion (PyObject *self, PyObject *args) { return Py_BuildValue("si", rd_kafka_version_str(), rd_kafka_version()); } static PyObject *version (PyObject *self, PyObject *args) { return Py_BuildValue("si", "0.9.2", 0x00090200); } static PyMethodDef cimpl_methods[] = { {"libversion", libversion, METH_NOARGS, " Retrieve librdkafka version string and integer\n" "\n" " :returns: (version_string, version_int) tuple\n" " :rtype: tuple(str,int)\n" "\n" }, {"version", version, METH_NOARGS, " Retrieve module version string and integer\n" "\n" " :returns: (version_string, version_int) tuple\n" " :rtype: tuple(str,int)\n" "\n" }, { NULL } }; /** * @brief Add librdkafka error enums to KafkaError's type dict. * @returns an updated doc string containing all error constants. */ static char *KafkaError_add_errs (PyObject *dict, const char *origdoc) { const struct rd_kafka_err_desc *descs; size_t cnt; size_t i; char *doc; size_t dof = 0, dsize; /* RST grid table column widths */ #define _COL1_W 50 #define _COL2_W 100 /* Must be larger than COL1 */ char dash[_COL2_W], eq[_COL2_W]; rd_kafka_get_err_descs(&descs, &cnt); memset(dash, '-', sizeof(dash)); memset(eq, '=', sizeof(eq)); /* Setup output doc buffer. */ dof = strlen(origdoc); dsize = dof + 500 + (cnt * 200); doc = malloc(dsize); memcpy(doc, origdoc, dof+1); #define _PRINT(...) do { \ char tmpdoc[512]; \ size_t _len; \ _len = snprintf(tmpdoc, sizeof(tmpdoc), __VA_ARGS__); \ if (_len > sizeof(tmpdoc)) _len = sizeof(tmpdoc)-1; \ if (dof + _len >= dsize) { \ dsize += 2; \ doc = realloc(doc, dsize); \ } \ memcpy(doc+dof, tmpdoc, _len+1); \ dof += _len; \ } while (0) /* Error constant table header (RST grid table) */ _PRINT("Error and event constants:\n\n" "+-%.*s-+-%.*s-+\n" "| %-*.*s | %-*.*s |\n" "+=%.*s=+=%.*s=+\n", _COL1_W, dash, _COL2_W, dash, _COL1_W, _COL1_W, "Constant", _COL2_W, _COL2_W, "Description", _COL1_W, eq, _COL2_W, eq); for (i = 0 ; i < cnt ; i++) { PyObject *code; if (!descs[i].desc) continue; code = PyLong_FromLong(descs[i].code); PyDict_SetItemString(dict, descs[i].name, code); Py_DECREF(code); _PRINT("| %-*.*s | %-*.*s |\n" "+-%.*s-+-%.*s-+\n", _COL1_W, _COL1_W, descs[i].name, _COL2_W, _COL2_W, descs[i].desc, _COL1_W, dash, _COL2_W, dash); } _PRINT("\n"); return doc; // FIXME: leak } #ifdef PY3 static struct PyModuleDef cimpl_moduledef = { PyModuleDef_HEAD_INIT, "cimpl", /* m_name */ "Confluent's Apache Kafka Python client (C implementation)", /* m_doc */ -1, /* m_size */ cimpl_methods, /* m_methods */ }; #endif static PyObject *_init_cimpl (void) { PyObject *m; if (PyType_Ready(&KafkaErrorType) < 0) return NULL; if (PyType_Ready(&MessageType) < 0) return NULL; if (PyType_Ready(&TopicPartitionType) < 0) return NULL; if (PyType_Ready(&ProducerType) < 0) return NULL; if (PyType_Ready(&ConsumerType) < 0) return NULL; #ifdef PY3 m = PyModule_Create(&cimpl_moduledef); #else m = Py_InitModule3("cimpl", cimpl_methods, "Confluent's Apache Kafka Python client (C implementation)"); #endif if (!m) return NULL; Py_INCREF(&KafkaErrorType); KafkaErrorType.tp_doc = KafkaError_add_errs(KafkaErrorType.tp_dict, KafkaErrorType.tp_doc); PyModule_AddObject(m, "KafkaError", (PyObject *)&KafkaErrorType); Py_INCREF(&MessageType); PyModule_AddObject(m, "Message", (PyObject *)&MessageType); Py_INCREF(&TopicPartitionType); PyModule_AddObject(m, "TopicPartition", (PyObject *)&TopicPartitionType); Py_INCREF(&ProducerType); PyModule_AddObject(m, "Producer", (PyObject *)&ProducerType); Py_INCREF(&ConsumerType); PyModule_AddObject(m, "Consumer", (PyObject *)&ConsumerType); KafkaException = PyErr_NewExceptionWithDoc( "cimpl.KafkaException", "Kafka exception that wraps the :py:class:`KafkaError` " "class.\n" "\n" "Use ``exception.args[0]`` to extract the " ":py:class:`KafkaError` object\n" "\n", NULL, NULL); Py_INCREF(KafkaException); PyModule_AddObject(m, "KafkaException", KafkaException); return m; } #ifdef PY3 PyMODINIT_FUNC PyInit_cimpl (void) { return _init_cimpl(); } #else PyMODINIT_FUNC initcimpl (void) { _init_cimpl(); } #endif python-confluent-kafka-0.9.2/confluent_kafka/src/confluent_kafka.h000066400000000000000000000140141303570112400253310ustar00rootroot00000000000000/** * Copyright 2016 Confluent Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #if PY_MAJOR_VERSION >= 3 #define PY3 #include #endif /**************************************************************************** * * * Python 2 & 3 portability * * Binary data (we call it cfl_PyBin): * Python 2: string * Python 3: bytes * * Unicode Strings (we call it cfl_PyUnistr): * Python 2: unicode * Python 3: strings * ****************************************************************************/ #ifdef PY3 /* Python 3 */ /** * @brief Binary type, use as cfl_PyBin(_X(A,B)) where _X() is the type-less * suffix of a PyBytes/Str_X() function */ #define cfl_PyBin(X) PyBytes ## X /** * @brief Unicode type, same usage as PyBin() */ #define cfl_PyUnistr(X) PyUnicode ## X /** * @returns Unicode Python object as char * in UTF-8 encoding */ #define cfl_PyUnistr_AsUTF8(X) PyUnicode_AsUTF8(X) /** * @returns Unicode Python string object */ #define cfl_PyObject_Unistr(X) PyObject_Str(X) #else /* Python 2 */ /* See comments above */ #define cfl_PyBin(X) PyString ## X #define cfl_PyUnistr(X) PyUnicode ## X #define cfl_PyUnistr_AsUTF8(X) PyBytes_AsString(PyUnicode_AsUTF8String(X)) #define cfl_PyObject_Unistr(X) PyObject_Unicode(X) #endif /**************************************************************************** * * * KafkaError * * * * ****************************************************************************/ extern PyObject *KafkaException; PyObject *KafkaError_new0 (rd_kafka_resp_err_t err, const char *fmt, ...); PyObject *KafkaError_new_or_None (rd_kafka_resp_err_t err, const char *str); /** * @brief Raise an exception using KafkaError. * \p err and and \p ... (string representation of error) is set on the returned * KafkaError object. */ #define cfl_PyErr_Format(err,...) do { \ PyObject *_eo = KafkaError_new0(err, __VA_ARGS__); \ PyErr_SetObject(KafkaException, _eo); \ } while (0) /**************************************************************************** * * * Common instance handle for both Producer and Consumer * * * * ****************************************************************************/ typedef struct { PyObject_HEAD rd_kafka_t *rk; PyObject *error_cb; int tlskey; /* Thread-Local-Storage key */ union { /** * Producer */ struct { PyObject *default_dr_cb; PyObject *partitioner_cb; /**< Registered Python partitioner */ int32_t (*c_partitioner_cb) ( const rd_kafka_topic_t *, const void *, size_t, int32_t, void *, void *); /**< Fallback C partitioner*/ } Producer; /** * Consumer */ struct { int rebalance_assigned; /* Rebalance: Callback performed assign() call.*/ PyObject *on_assign; /* Rebalance: on_assign callback */ PyObject *on_revoke; /* Rebalance: on_revoke callback */ PyObject *on_commit; /* Commit callback */ } Consumer; } u; } Handle; void Handle_clear (Handle *h); int Handle_traverse (Handle *h, visitproc visit, void *arg); /** * @brief Current thread's state for "blocking" calls to librdkafka. */ typedef struct { PyThreadState *thread_state; int crashed; /* Callback crashed */ } CallState; /** * @brief Initialiase a CallState and unlock the GIL prior to a * possibly blocking external call. */ void CallState_begin (Handle *h, CallState *cs); /** * @brief Relock the GIL after external call is done, remove TLS state. * @returns 0 if a Python signal was raised or a callback crashed, else 1. */ int CallState_end (Handle *h, CallState *cs); /** * @brief Get the current thread's CallState and re-locks the GIL. */ CallState *CallState_get (Handle *h); /** * @brief Un-locks the GIL to resume blocking external call. */ void CallState_resume (CallState *cs); /** * @brief Indicate that call crashed. */ void CallState_crash (CallState *cs); /**************************************************************************** * * * Common * * * * ****************************************************************************/ rd_kafka_conf_t *common_conf_setup (rd_kafka_type_t ktype, Handle *h, PyObject *args, PyObject *kwargs); PyObject *c_parts_to_py (const rd_kafka_topic_partition_list_t *c_parts); rd_kafka_topic_partition_list_t *py_to_c_parts (PyObject *plist); /**************************************************************************** * * * Message * * * * ****************************************************************************/ /** * @brief confluent_kafka.Message object */ typedef struct { PyObject_HEAD PyObject *topic; PyObject *value; PyObject *key; PyObject *error; int32_t partition; int64_t offset; } Message; extern PyTypeObject MessageType; PyObject *Message_new0 (const rd_kafka_message_t *rkm); PyObject *Message_error (Message *self, PyObject *ignore); /**************************************************************************** * * * Producer * * * * ****************************************************************************/ extern PyTypeObject ProducerType; int32_t Producer_partitioner_cb (const rd_kafka_topic_t *rkt, const void *keydata, size_t keylen, int32_t partition_cnt, void *rkt_opaque, void *msg_opaque); /**************************************************************************** * * * Consumer * * * * ****************************************************************************/ extern PyTypeObject ConsumerType; python-confluent-kafka-0.9.2/setup.cfg000066400000000000000000000000731303570112400177260ustar00rootroot00000000000000[egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 python-confluent-kafka-0.9.2/setup.py000077500000000000000000000013571303570112400176300ustar00rootroot00000000000000#!/usr/bin/env python from setuptools import setup, find_packages from distutils.core import Extension module = Extension('confluent_kafka.cimpl', libraries= ['rdkafka'], sources=['confluent_kafka/src/confluent_kafka.c', 'confluent_kafka/src/Producer.c', 'confluent_kafka/src/Consumer.c']) setup(name='confluent-kafka', version='0.9.2', description='Confluent\'s Apache Kafka client for Python', author='Confluent Inc', author_email='support@confluent.io', url='https://github.com/confluentinc/confluent-kafka-python', ext_modules=[module], packages=find_packages(), data_files = [('', ['LICENSE'])]) python-confluent-kafka-0.9.2/tests/000077500000000000000000000000001303570112400172475ustar00rootroot00000000000000python-confluent-kafka-0.9.2/tests/test_Consumer.py000066400000000000000000000032331303570112400224540ustar00rootroot00000000000000#!/usr/bin/env python from confluent_kafka import Consumer, TopicPartition, KafkaError, KafkaException def test_basic_api(): """ Basic API tests, these wont really do anything since there is no broker configured. """ try: kc = Consumer() except TypeError as e: assert str(e) == "expected configuration dict" def dummy_commit_cb (err, partitions): pass kc = Consumer({'group.id':'test', 'socket.timeout.ms':'100', 'session.timeout.ms': 1000, # Avoid close() blocking too long 'on_commit': dummy_commit_cb}) kc.subscribe(["test"]) kc.unsubscribe() def dummy_assign_revoke (consumer, partitions): pass kc.subscribe(["test"], on_assign=dummy_assign_revoke, on_revoke=dummy_assign_revoke) kc.unsubscribe() msg = kc.poll(timeout=0.001) if msg is None: print('OK: poll() timeout') elif msg.error(): print('OK: consumer error: %s' % msg.error().str()) else: print('OK: consumed message') partitions = list(map(lambda p: TopicPartition("test", p), range(0,100,3))) kc.assign(partitions) kc.unassign() kc.commit(async=True) try: kc.commit(async=False) except KafkaException as e: assert e.args[0].code() in (KafkaError._TIMED_OUT, KafkaError._NO_OFFSET) # Get current position, should all be invalid. kc.position(partitions) assert len([p for p in partitions if p.offset == -1001]) == len(partitions) try: offsets = kc.committed(partitions, timeout=0.001) except KafkaException as e: assert e.args[0].code() == KafkaError._TIMED_OUT kc.close() python-confluent-kafka-0.9.2/tests/test_KafkaError.py000066400000000000000000000014151303570112400227100ustar00rootroot00000000000000#!/usr/bin/env python from confluent_kafka import Producer, KafkaError, KafkaException import time seen_all_brokers_down = False def error_cb (err): print('error_cb', err) if err.code() == KafkaError._ALL_BROKERS_DOWN: global seen_all_brokers_down seen_all_brokers_down = True def test_error_cb(): """ Test the error callback. """ global seen_all_brokers_down # Configure an invalid broker and make sure the ALL_BROKERS_DOWN # error is seen in the error callback. p = Producer({'bootstrap.servers': '127.0.0.1:1', 'socket.timeout.ms':10, 'error_cb': error_cb}) t_end = time.time() + 5 while not seen_all_brokers_down and time.time() < t_end: p.poll(1) assert seen_all_brokers_down python-confluent-kafka-0.9.2/tests/test_Producer.py000066400000000000000000000016701303570112400224470ustar00rootroot00000000000000#!/usr/bin/env python from confluent_kafka import Producer, KafkaError, KafkaException def test_basic_api(): """ Basic API tests, these wont really do anything since there is no broker configured. """ try: p = Producer() except TypeError as e: assert str(e) == "expected configuration dict" def error_cb (err): print('error_cb', err) p = Producer({'socket.timeout.ms':10, 'error_cb': error_cb, 'default.topic.config': {'message.timeout.ms': 10}}) p.produce('mytopic') p.produce('mytopic', value='somedata', key='a key') def on_delivery(err,msg): print('delivery', str) # Since there is no broker, produced messages should time out. assert err.code() == KafkaError._MSG_TIMED_OUT p.produce(topic='another_topic', value='testing', partition=9, callback=on_delivery) p.poll(0.001) p.flush() python-confluent-kafka-0.9.2/tests/test_TopicPartition.py000066400000000000000000000021741303570112400236340ustar00rootroot00000000000000#!/usr/bin/env python from confluent_kafka import TopicPartition def test_sort(): """ TopicPartition sorting (rich comparator) """ # sorting uses the comparator correct = [TopicPartition('topic1', 3), TopicPartition('topic3', 0), TopicPartition('topicA', 5), TopicPartition('topicA', 5)] tps = sorted([TopicPartition('topicA', 5), TopicPartition('topic3', 0), TopicPartition('topicA', 5), TopicPartition('topic1', 3)]) assert correct == tps def test_cmp(): """ TopicPartition comparator """ assert TopicPartition('aa', 19002) > TopicPartition('aa', 0) assert TopicPartition('aa', 13) >= TopicPartition('aa', 12) assert TopicPartition('BaB', 9) != TopicPartition('Card', 9) assert TopicPartition('b3x', 4) == TopicPartition('b3x', 4) assert TopicPartition('ulv', 2) < TopicPartition('xy', 0) assert TopicPartition('ulv', 2) <= TopicPartition('ulv', 3) def test_hash(): tp1 = TopicPartition('test', 99) tp2 = TopicPartition('somethingelse', 12) assert hash(tp1) != hash(tp2) python-confluent-kafka-0.9.2/tests/test_docs.py000066400000000000000000000013171303570112400216120ustar00rootroot00000000000000#!/usr/bin/env python import confluent_kafka import re from types import ModuleType def test_verify_docs(): """ Make sure all exported functions, classes, etc, have proper docstrings """ fails = 0 for n in dir(confluent_kafka): if n.startswith('__'): # Skip internals continue o = confluent_kafka.__dict__.get(n) d = o.__doc__ if not d: print('Missing __doc__ for: %s (type %s)' % (n, type(o))) fails += 1 elif not re.search(r':', d): print('Missing Doxygen tag for: %s (type %s)' % (n, type(o))) if not isinstance(o, ModuleType): fails += 1 assert fails == 0 python-confluent-kafka-0.9.2/tests/test_enums.py000066400000000000000000000004641303570112400220130ustar00rootroot00000000000000#!/usr/bin/env python import confluent_kafka def test_enums(): """ Make sure librdkafka error enums are reachable directly from the KafkaError class without an instantiated object. """ print(confluent_kafka.KafkaError._NO_OFFSET) print(confluent_kafka.KafkaError.REBALANCE_IN_PROGRESS) python-confluent-kafka-0.9.2/tests/test_misc.py000066400000000000000000000006401303570112400216130ustar00rootroot00000000000000#!/usr/bin/env python import confluent_kafka def test_version(): print('Using confluent_kafka module version %s (0x%x)' % confluent_kafka.version()) sver, iver = confluent_kafka.version() assert len(sver) > 0 assert iver > 0 print('Using librdkafka version %s (0x%x)' % confluent_kafka.libversion()) sver, iver = confluent_kafka.libversion() assert len(sver) > 0 assert iver > 0 python-confluent-kafka-0.9.2/tests/test_threads.py000066400000000000000000000030501303570112400223100ustar00rootroot00000000000000#!/usr/bin/env python from confluent_kafka import Producer, KafkaError, KafkaException import threading import time try: from queue import Queue, Empty except: from Queue import Queue, Empty class IntendedException (Exception): pass def thread_run(myid, p, q): def do_crash(err, msg): raise IntendedException() for i in range(1, 3): cb = None if i == 2: cb = do_crash p.produce('mytopic', value='hi', callback=cb) t = time.time() try: p.flush() print(myid, 'Flush took %.3f' % (time.time() - t)) except IntendedException: print(myid, "Intentional callback crash: ok") continue print(myid, 'Done') q.put(myid) def test_thread_safety(): """ Basic thread safety tests. """ q = Queue() p = Producer({'socket.timeout.ms': 10, 'socket.blocking.max.ms': 10, 'default.topic.config': {'message.timeout.ms': 10}}) threads = list() for i in range(1, 5): thr = threading.Thread(target=thread_run, name=str(i), args=[i, p, q]) thr.start() threads.append(thr) for thr in threads: thr.join() # Count the number of threads that exited cleanly cnt = 0 try: for x in iter(q.get_nowait, None): cnt += 1 except Empty: pass if cnt != len(threads): raise Exception('Only %d/%d threads succeeded' % (cnt, len(threads))) print('Done') if __name__ == '__main__': test_thread_safety()