pax_global_header00006660000000000000000000000064130346615210014513gustar00rootroot0000000000000052 comment=6d4882c12b3f54da27a54ec92f4b7f9882cba3a4 carbon-c-relay-2.5/000077500000000000000000000000001303466152100141575ustar00rootroot00000000000000carbon-c-relay-2.5/.gitignore000066400000000000000000000000131303466152100161410ustar00rootroot00000000000000*.o /relay carbon-c-relay-2.5/.travis.yml000066400000000000000000000025371303466152100162770ustar00rootroot00000000000000script: make language: c os: - linux - osx compiler: - gcc - clang matrix: exclude: # gcc is actually clang on OSX, so pointless to test twice - os: osx compiler: gcc env: global: # The next declaration is the encrypted COVERITY_SCAN_TOKEN, created # via the "travis encrypt" command using the project repo's public key - secure: "H844xDJC5CUXhUyB1SFsGiiz/Q4viVSALkj/KnWAuagrbrzIXJoK4tRWpPxCzUar5QsdEua1WIXTBezPA6V97w2cis2OGUXtgsMd1fWnwlG0RwPRj2+LAVxJ+i5z1Asx8hkYOiZIv8y0VkBGldjOrD0l0pBcxB+CDZC/zsGSRtnMLOs+smfEjOF6JPGw8Ye0XJ+YKrfbHOXhfS7XTMwUnjgAV7lTH/aAlKuedc3mYsuC3WHJOMptYLXaiPdbPKoIXfLdmszP91SkThr4f0elyQskugzGM3D7M1tIFhfmordSD+DfzLvvdSu32hHUNQjHvAm7Pz8QmdLC2IjjE8y8MkIkEy2+wR/5AjacnKR8OgFzXgt+yIH+CBc6qbTi40dZ7Go33Q8uNyKJGrA1CyL195NS8q0reMbuS7WIxI0rNOy7cdw8MO0sNjeofcb1RW90U5u5qciRQpbwhFwMFMCuD0LL41mjfN+IHGNeUwp148Qibbvyu+d7y4Y1F7/CsdMpZVBH9Ae7CHXetN8biT81fJp1rn0uEGQqoPuxvKeePsL40MXu6wvlrSf5wLKV3aqp5lzbjoJsPrOwV5iVwMHrvdhoAFlLc6Y1aJkCTPPt7yjLHgLBisqRXLTZ9lcD3AG8ubTLr03DsseUUM3/tWvhWfVRfxzkuOHsDRLMfdY2Q20=" - CFLAGS="-O3 -Wall -Werror -Wshadow -pipe" addons: coverity_scan: project: name: "grobian/carbon-c-relay" description: "Build submitted via Travis CI" notification_email: grobian@gentoo.org build_command_prepend: "make clean" build_command: "make -j 4" branch_pattern: master carbon-c-relay-2.5/ChangeLog.md000066400000000000000000000225151303466152100163350ustar00rootroot00000000000000# 2.6 (unreleased master branch) # 2.5 (2017-01-09) ### Bugfixes * [Issue #239](https://github.com/grobian/carbon-c-relay/issues/239) segfault when date format is incorrect * [Issue #242](https://github.com/grobian/carbon-c-relay/issues/242) dispatcher/aggregations broken (relay seems too slow) # 2.4 (2017-01-03) ### New Features * **router** `match` rules now support a `validate` clause to do data filtering, [Issue #228](https://github.com/grobian/carbon-c-relay/issues/228), [Issue #142](https://github.com/grobian/carbon-c-relay/issues/142), [Issue #121](https://github.com/grobian/carbon-c-relay/issues/121), [Pull #127](https://github.com/grobian/carbon-c-relay/issues/127), [Pull #87](https://github.com/grobian/carbon-c-relay/issues/87). ### Bugfixes * **server** connection errors are no longer endlessly repeated * [Issue #240](https://github.com/grobian/carbon-c-relay/issues/240) 'include' directive doesn't care about rewrites. * [Issue #241](https://github.com/grobian/carbon-c-relay/issues/241) XXX characters being prepended to metrics when sent via UDP * [Issue #204](https://github.com/grobian/carbon-c-relay/issues/204) relay is sending data randomly while kill -HUP happens # 2.3 (2016-11-07) ### Bugfixes * [Issue #213](https://github.com/grobian/carbon-c-relay/issues/213) Change to aggregates to not cause HUP to reload when more than 10 aggregates are defined. * [Issue #214](https://github.com/grobian/carbon-c-relay/issues/214) `-U` option doesn't set UDP receive buffer size. * [Issue #218](https://github.com/grobian/carbon-c-relay/issues/218) zeros inserted after some metrics. * [Issue #219](https://github.com/grobian/carbon-c-relay/issues/219) should fail if port is unavailable * [Issue #224](https://github.com/grobian/carbon-c-relay/issues/224) segfault during SIGHUP # 2.2 (2016-09-11) ### New Features * **relay** socket receive and send buffer sizes can be adjusted the `-U` option was introduced to allow setting the socket buffer size in bytes, [Issue #207](https://github.com/grobian/carbon-c-relay/issues/207). ### Breaking Changes ### Enhancements * **relay** the listen backlog default got increased from 3 to 32 * **server** TCP\_NODELAY is now set to improve small writes ### Bugfixes * [Issue #188](https://github.com/grobian/carbon-c-relay/issues/188) SIGHUP leads to SIGSEGV when config didn't change, or a SIGHUP is received while a previous HUP is being processed * [Issue #193](https://github.com/grobian/carbon-c-relay/issues/193) race condition in aggregator leads to crash * [Issue #195](https://github.com/grobian/carbon-c-relay/issues/195) assertion fails when reloading config * [Issue #200](https://github.com/grobian/carbon-c-relay/issues/200) use after free during shutdown in aggregator * [Issue #203](https://github.com/grobian/carbon-c-relay/issues/203) change default connection listen backlog * [Issue #208](https://github.com/grobian/carbon-c-relay/issues/208) TCP\_NODELAY should be off for connections relaying data * [Issue #199](https://github.com/grobian/carbon-c-relay/issues/199) Various race conditions reported by TSAN # 2.1 (16-06-2016) ### Enhancements * **router** the optimiser now tries harder to form groups of consecutive rules that have a matching common pattern ### Bugfixes * [Issue #180](https://github.com/grobian/carbon-c-relay/issues/180) include directive possibly overrides previous included components * [Issue #184](https://github.com/grobian/carbon-c-relay/issues/184) router optimise doesn't work correctly with regex groups # 2.0 (30-05-2016) ### New Features * **router** `include` directive was added to add content of another file, see also [Issue #165](https://github.com/grobian/carbon-c-relay/issues/165). The include can also use glob patterns, see [Pull #174](https://github.com/grobian/carbon-c-relay/pull/174) * **server** the number of stalls performed on writes can now be controlled (and also disabled) using the `-L` flag. [Issue #172](https://github.com/grobian/carbon-c-relay/issues/172) ### Breaking Changes ### Enhancements * **server** incomplete writes are now retried a couple of times before they are considered fatal. This should reduce the amount of messages in the logs about them, and be more like the consumer expects, e.g. less sudden disconnects for the client. * **router** reloading the config now prints the difference between the old and the new config in `diff -u` format. * **router** reloading the config now maintains the queues for the servers, such that unavailable servers don't get metrics dropped. ### Bugfixes * [Issue #154](https://github.com/grobian/carbon-c-relay/issues/159) when a store becomes a bottleneck it shouldn't indefinitely stall * [Issue #164](https://github.com/grobian/carbon-c-relay/issues/164) config reload should re-use unmodified servers # 1.11 (23-03-2016) ### New Features * **router** `send statistics to` construct was added to direct internal statistics to a specific cluster ### Enhancements * **collector** UDP connections are now suffixed with `-udp` in destination target ### Bugfixes * [Issue #159](https://github.com/grobian/carbon-c-relay/issues/159) corrupted statistics for file clusters * [Issue #160](https://github.com/grobian/carbon-c-relay/issues/160) metricsBlackholed stays zero when blackhole target is used # 1.10 (09-03-2016) ### Breaking Changes * **statistics** dispatch\_busy and dispatch\_idle have been replaced with wallTime\_us and sleepTime\_us ### Bugfixes * [Issue #152](https://github.com/grobian/carbon-c-relay/issues/152) crash in aggregator\_expire for data-contained aggregations # 1.9 (07-03-2016) ### Enhancements * **statistics** dispatch\_busy is slightly more realistic now ### Bugfixes * [Issue #153](https://github.com/grobian/carbon-c-relay/issues/153) aggregator statistics are garbage with `-m` # 1.8 (23-02-2016) ### New Features * **relay** new flags `-D` for daemon mode and `-p` for pidfile creation ### Enhancements * **dispatcher** server stalling (to slow down too fast writers) is now based on a random timeout * **server** write timeout is now large enough to deal with upstream relay stalling * **relay** number of workers/dispatchers is now determined in a way that doesn''t need OpenMP any more # 1.7 (29-01-2016) ### New Features * **relay** new flag `-B` to set the listen backlog for TCP and UNIX connections, [issue #143](https://github.com/grobian/carbon-c-relay/issues/143) ### Enhancements * **dispatcher** switch from select() to poll() to fix crashes when too many connections were made to the relay * Misc (memory) leak fixes # 1.6 (27-01-2016) ### Breaking Changes * **relay** startup and shutdown messages are now better in line ### Enhancements * **relay** fixed segfault when issuing `SIGHUP` under active load # 1.5 (13-01-2016) ### Enhancements * **aggregator** metrics are now written directly to dispatchers to avoid overload of the internal\_submission queue, which is likely to to happen with many aggregates * **collector** properly report file-based servers in statistics * **collector** re-introduce the interal destination in statistics # 1.4 (04-01-2016) ### New Features * **collector** when run in debug and submission mode, there is a iostat like output ### Enhancements * **relay** reloading config now no longer unconditionally starts the aggregator * **aggregator** misc cleanup/free fixes * **relay** allow reloading aggregator ### Bugfixes * [Issue #133](https://github.com/grobian/carbon-c-relay/issues/133) _stub_aggregator metrics seen after a reload # 1.3 (16-12-2015) ### Enhancements * **consistent-hash** fix jump\_fnv1a\_ch metric submission, it didn''t work at all ### Bugfixes * [Issue #126](https://github.com/grobian/carbon-c-relay/issues/126) double free crash * [Issue #131](https://github.com/grobian/carbon-c-relay/issues/131) segfault using stddev in aggregator * [Issue #132](https://github.com/grobian/carbon-c-relay/issues/132) crash with glibc double free message # 1.2 (10-12-2015) ### New Features * **consistent-hash** new algorithm jump\_fnv1a\_ch for near perfect distribution of metrics * **distributiontest** test program used to see unbalancedness of clusters for a given input metric see [graphite-project/carbon#485](https://github.com/graphite-project/carbon/issues/485) ### Enhancements * **router** fix cluster checking with regards replication count and the number of servers to allow equal counts ### Bugfixes * [Issue #126](https://github.com/grobian/carbon-c-relay/issues/126) prevent calling read() too often # 1.1 (25-11-2015) ### Enhancements * **router** fix distribution of any\_of cluster if members have failed # 1.0 (23-11-2015) * many improvements # 0.45 (05-11-2015) * Many aggregator improvements, more flexible routing support. # 0.44 (13-08-2015) * Feature to set hash-keys for fnv1a\_ch. # 0.43 (27-07-2015) * Bugfix release for segfault when using any\_of clusters. # 0.42 (24-07-2015) * Reduced warning level for submission mode queue pileups. Allow writing to a file (cluster type). Fix splay on aggregator not to affect timestamps of input. No more dep on openssl for md5. # 0.40 (11-05-2015) * Hefty optimisations on aggregations. Fix for UDP port closure. carbon-c-relay-2.5/LICENSE.md000066400000000000000000000236761303466152100156010ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS carbon-c-relay-2.5/Makefile000066400000000000000000000031201303466152100156130ustar00rootroot00000000000000# Copyright 2013-2017 Fabian Groffen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. CFLAGS ?= -O2 -Wall -Wshadow GIT_VERSION := $(shell git describe --abbrev=6 --dirty --always 2>/dev/null || date +%F) GVCFLAGS += -DGIT_VERSION=\"$(GIT_VERSION)\" override CFLAGS += $(GVCFLAGS) -pthread SOCKET_LIBS = ifeq ($(shell uname), SunOS) SOCKET_LIBS += -lsocket -lnsl endif # should be accepted sort of anywhere MATH_LIBS = -lm override LIBS += $(SOCKET_LIBS) $(MATH_LIBS) -pthread OBJS = \ relay.o \ md5.o \ consistent-hash.o \ receptor.o \ dispatcher.o \ router.o \ queue.o \ server.o \ collector.o \ aggregator.o relay: $(OBJS) $(CC) -o $@ $(LDFLAGS) $^ $(LIBS) man: sed -e '/travis-ci.org\/grobian\/carbon-c-relay.svg/d' carbon-c-relay.md | \ ronn \ --manual="Graphite data collection and visualisation" \ --organization=Graphite \ --roff \ > carbon-c-relay.1 VERSION = $(shell sed -n '/VERSION/s/^.*"\([0-9.]\+\)".*$$/\1/p' relay.h) dist: git archive \ --format=tar.gz \ --prefix=carbon-c-relay-$(VERSION)/ v$(VERSION) \ > carbon-c-relay-$(VERSION).tar.gz clean: rm -f *.o relay carbon-c-relay-2.5/README.md000077700000000000000000000000001303466152100205512carbon-c-relay.mdustar00rootroot00000000000000carbon-c-relay-2.5/aggregator.c000066400000000000000000000443401303466152100164520ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include "relay.h" #include "dispatcher.h" #include "server.h" #include "router.h" #include "aggregator.h" #include "fnv1a.h" static pthread_t aggregatorid; static size_t prevreceived = 0; static size_t prevsent = 0; static size_t prevdropped = 0; static char keep_running = 1; /** * Allocates a new aggregator setup to hold buckets matching interval * and expiry time. */ aggregator * aggregator_new( unsigned int interval, unsigned int expire, enum _aggr_timestamp tswhen) { aggregator *ret = malloc(sizeof(aggregator)); int intconn[2]; if (ret == NULL) return ret; assert(interval != 0); assert(interval < expire); if (pipe(intconn) < 0) { logerr("failed to create pipe for aggregator: %s\n", strerror(errno)); free(ret); return NULL; } ret->disp_conn = dispatch_addconnection_aggr(intconn[0]); ret->fd = intconn[1]; ret->interval = interval; ret->expire = expire; ret->tswhen = tswhen; ret->bucketcnt = (expire + (interval - 1)) / interval + 1 + 1; ret->received = 0; ret->sent = 0; ret->dropped = 0; ret->computes = NULL; ret->next = NULL; return ret; } /** * Adds a new compute part to this aggregator. Returns -1 if type is * not a recognised aggregation type. */ char aggregator_add_compute( aggregator *s, const char *metric, const char *type) { struct _aggr_computes *ac = s->computes; enum _aggr_compute_type act; char store = 0; int pctl = 0; if (strcmp(type, "sum") == 0) { act = SUM; } else if (strcmp(type, "count") == 0 || strcmp(type, "cnt") == 0) { act = CNT; } else if (strcmp(type, "max") == 0) { act = MAX; } else if (strcmp(type, "min") == 0) { act = MIN; } else if (strcmp(type, "average") == 0 || strcmp(type, "avg") == 0) { act = AVG; } else if (strcmp(type, "median") == 0) { act = MEDN; pctl = 50; store = 1; } else if (strncmp(type, "percentile", strlen("percentile")) == 0) { pctl = atoi(type + strlen("percentile")); if (pctl > 100 || pctl <= 0) { return -1; } else { act = PCTL; store = 1; } } else if (strcmp(type, "variance") == 0) { act = VAR; store = 1; } else if (strcmp(type, "stddev") == 0) { act = SDEV; store = 1; } else { return -1; } if (ac == NULL) { ac = s->computes = malloc(sizeof(*ac)); } else { while (ac->next != NULL) ac = ac->next; ac = ac->next = malloc(sizeof(*ac)); } ac->type = act; ac->percentile = (unsigned char)pctl; ac->metric = strdup(metric); memset(ac->invocations_ht, 0, sizeof(ac->invocations_ht)); ac->entries_needed = store; pthread_rwlock_init(&ac->invlock, NULL); ac->next = NULL; return 0; } void aggregator_set_stub( aggregator *s, const char *stubname) { struct _aggr_computes *ac; char newmetric[METRIC_BUFSIZ]; for (ac = s->computes; ac != NULL; ac = ac->next) { snprintf(newmetric, sizeof(newmetric), "%s%s", stubname, ac->metric); free((void *)ac->metric); ac->metric = strdup(newmetric); } } /** * Adds a new metric to aggregator s. The value from the metric is put * in the bucket matching the epoch contained in the metric. In cases * where the contained epoch is too old or too new, the metric is * dropped. */ void aggregator_putmetric( aggregator *s, const char *metric, const char *firstspace, size_t nmatch, regmatch_t *pmatch) { char *v; double val; long long int epoch; long long int itime; char newmetric[METRIC_BUFSIZ]; char *newfirstspace = NULL; size_t len; const char *ometric; const char *omp; unsigned int omhash; unsigned int omhtbucket; struct _aggr_computes *compute; struct _aggr_invocations *invocation; struct _aggr_bucket *bucket; struct _aggr_bucket_entries *entries; /* do not accept new values when shutting down, issue #200 */ if (__sync_bool_compare_and_swap(&keep_running, 0, 0)) return; /* get timestamp */ if ((v = strchr(firstspace + 1, ' ')) == NULL) { /* metric includes \n */ if (mode & MODE_DEBUG) logerr("aggregator: dropping incorrect metric: %s", metric); return; } __sync_add_and_fetch(&s->received, 1); val = atof(firstspace + 1); epoch = atoll(v + 1); for (compute = s->computes; compute != NULL; compute = compute->next) { if (nmatch == 0) { ometric = compute->metric; } else if ((len = router_rewrite_metric( &newmetric, &newfirstspace, metric, firstspace, compute->metric, nmatch, pmatch)) == 0) { /* fail, skip */ continue; } else { *newfirstspace = '\0'; ometric = newmetric; } omhash = FNV1A_32_OFFSET; for (omp = ometric; *omp != '\0'; omp++) omhash = (omhash ^ (unsigned int)*omp) * FNV1A_32_PRIME; omhtbucket = ((omhash >> AGGR_HT_POW_SIZE) ^ omhash) & (((unsigned int)1 << AGGR_HT_POW_SIZE) - 1); pthread_rwlock_wrlock(&compute->invlock); invocation = compute->invocations_ht[omhtbucket]; for (; invocation != NULL; invocation = invocation->next) if (invocation->hash == omhash && strcmp(ometric, invocation->metric) == 0) /* match */ break; if (invocation == NULL) { /* no match, add */ long long int i; time_t now; if ((invocation = malloc(sizeof(*invocation))) == NULL) { logerr("aggregator: out of memory creating %s from %s", ometric, metric); pthread_rwlock_unlock(&compute->invlock); continue; } if ((invocation->metric = strdup(ometric)) == NULL) { logerr("aggregator: out of memory creating %s from %s", ometric, metric); free(invocation); pthread_rwlock_unlock(&compute->invlock); continue; } invocation->hash = omhash; /* Start buckets in the past such that expiry time * conditions are met. Add a splay to the expiry time to * avoid a thundering herd of expirations when the * aggregator is spammed with metrics, e.g. right after * startup when other relays flush their queues. This * approach shouldn't affect the timing of the buckets as * requested in issue #72. * For consistency with other tools/old carbon-aggregator * align the buckets to interval boundaries such that it is * predictable what intervals will be taken, issue #104. */ time(&now); now = ((now - s->expire) / s->interval) * s->interval; invocation->expire = s->expire + (rand() % s->interval); /* allocate enough buckets to hold the past + future */ invocation->buckets = malloc(sizeof(struct _aggr_bucket) * s->bucketcnt); if (invocation->buckets == NULL) { logerr("aggregator: out of memory creating %s from %s", ometric, metric); free(invocation->metric); free(invocation); pthread_rwlock_unlock(&compute->invlock); continue; } for (i = 0; i < s->bucketcnt; i++) { invocation->buckets[i].start = now + (i * (long long int)s->interval); invocation->buckets[i].cnt = 0; invocation->buckets[i].entries.size = 0; invocation->buckets[i].entries.values = NULL; } invocation->next = compute->invocations_ht[omhtbucket]; compute->invocations_ht[omhtbucket] = invocation; } /* finally, try to do the maths */ itime = epoch - invocation->buckets[0].start; if (itime < 0) { /* drop too old metric */ __sync_add_and_fetch(&s->dropped, 1); pthread_rwlock_unlock(&compute->invlock); continue; } if (itime >= (s->bucketcnt * s->interval)) { if (mode & MODE_DEBUG) logerr("aggregator: dropping metric too far in the " "future (%lld > %lld): %s from %s", epoch, invocation->buckets[s->bucketcnt - 1].start, ometric, metric); __sync_add_and_fetch(&s->dropped, 1); pthread_rwlock_unlock(&compute->invlock); continue; } bucket = &invocation->buckets[itime / s->interval]; if (bucket->cnt == 0) { bucket->sum = val; bucket->max = val; bucket->min = val; } else { bucket->sum += val; if (bucket->max < val) bucket->max = val; if (bucket->min > val) bucket->min = val; } entries = &bucket->entries; if (compute->entries_needed) { if (bucket->cnt == entries->size) { #define E_I_SZ 64 double *new = realloc(entries->values, sizeof(double) * (entries->size + E_I_SZ)); if (new == NULL) { logerr("aggregator: out of memory creating entry bucket " "(%s from %s)", ometric, metric); } else { entries->values = new; entries->size += E_I_SZ; } } if (bucket->cnt < entries->size) entries->values[bucket->cnt] = val; } bucket->cnt++; pthread_rwlock_unlock(&compute->invlock); } return; } static inline int cmp_entry(const void *l, const void *r) { return *(const double *)l - *(const double *)r; } /** * Checks if the oldest bucket should be expired, if so, sends out * computed aggregate metrics and moves the bucket to the end of the * list. When no buckets are in use for an invocation, it is removed to * cleanup resources. */ static void * aggregator_expire(void *sub) { aggregator *aggrs = (aggregator *)sub; time_t now; aggregator *s; struct _aggr_bucket *b; struct _aggr_computes *c; struct _aggr_invocations *inv; struct _aggr_invocations *lastinv; double *values; size_t len = 0; int i; size_t k; unsigned char j; int work; char metric[METRIC_BUFSIZ]; char isempty; long long int ts = 0; while (1) { work = 0; for (s = aggrs; s != NULL; s = s->next) { /* send metrics for buckets that are completely past the * expiry time, unless we are shutting down, then send * metrics for all buckets that have completed */ now = time(NULL) + (__sync_bool_compare_and_swap(&keep_running, 1, 1) ? 0 : s->expire - s->interval); for (c = s->computes; c != NULL; c = c->next) { pthread_rwlock_wrlock(&c->invlock); for (i = 0; i < (1 << AGGR_HT_POW_SIZE); i++) { lastinv = NULL; isempty = 0; for (inv = c->invocations_ht[i]; inv != NULL; ) { while (inv->buckets[0].start + (__sync_bool_compare_and_swap(&keep_running, 1, 1) ? inv->expire : s->expire) < now) { /* yay, let's produce something cool */ b = &inv->buckets[0]; /* avoid emitting empty/unitialised data */ isempty = b->cnt == 0; if (!isempty) { switch (s->tswhen) { case TS_START: ts = b->start; break; case TS_MIDDLE: ts = b->start + (s->interval / 2); break; case TS_END: ts = b->start + s->interval; break; default: assert(0); } switch (c->type) { case SUM: len = snprintf(metric, sizeof(metric), "%s %f %lld\n", inv->metric, b->sum, ts); break; case CNT: len = snprintf(metric, sizeof(metric), "%s %zu %lld\n", inv->metric, b->cnt, ts); break; case MAX: len = snprintf(metric, sizeof(metric), "%s %f %lld\n", inv->metric, b->max, ts); break; case MIN: len = snprintf(metric, sizeof(metric), "%s %f %lld\n", inv->metric, b->min, ts); break; case AVG: len = snprintf(metric, sizeof(metric), "%s %f %lld\n", inv->metric, b->sum / (double)b->cnt, ts); break; case MEDN: /* median == 50th percentile */ case PCTL: /* nearest rank method */ k = (int)(((double)c->percentile/100.0 * (double)b->cnt) + 0.9); values = b->entries.values; /* TODO: lazy approach, in case * of 1 (first/last) or 2 buckets * distance we could do a * forward run picking the max * entries and returning that * iso sorting the full array */ qsort(values, b->cnt, sizeof(double), cmp_entry); len = snprintf(metric, sizeof(metric), "%s %f %lld\n", inv->metric, values[k - 1], ts); break; case VAR: case SDEV: { double avg = b->sum / (double)b->cnt; double ksum = 0; values = b->entries.values; for (k = 0; k < b->cnt; k++) ksum += pow(values[k] - avg, 2); ksum /= (double)b->cnt; len = snprintf(metric, sizeof(metric), "%s %f %lld\n", inv->metric, c->type == VAR ? ksum : sqrt(ksum), ts); } break; default: assert(0); /* for compiler (len) */ } ts = write(s->fd, metric, len); if (ts < 0) { logerr("aggregator: failed to write to " "pipe (fd=%d): %s\n", s->fd, strerror(errno)); __sync_add_and_fetch(&s->dropped, 1); } else if (ts < len) { logerr("aggregator: uncomplete write on " "pipe (fd=%d)\n", s->fd); __sync_add_and_fetch(&s->dropped, 1); } else { __sync_add_and_fetch(&s->sent, 1); } } /* move the bucket to the end, to make room for * new ones */ b = &inv->buckets[0]; len = b->entries.size; values = b->entries.values; memmove(b, &inv->buckets[1], sizeof(*b) * (s->bucketcnt - 1)); b = &inv->buckets[s->bucketcnt - 1]; b->cnt = 0; b->start = inv->buckets[s->bucketcnt - 2].start + s->interval; b->entries.size = len; b->entries.values = values; work++; } if (isempty) { /* see if the remaining buckets are empty too */ for (j = 0; j < s->bucketcnt; j++) { if (inv->buckets[j].cnt != 0) { isempty = 0; break; } } } if (isempty) { /* free and unlink */ if (c->entries_needed) for (j = 0; j < s->bucketcnt; j++) if (inv->buckets[j].entries.values) free(inv->buckets[j].entries.values); free(inv->metric); free(inv->buckets); if (lastinv != NULL) { lastinv->next = inv->next; free(inv); inv = lastinv->next; } else { c->invocations_ht[i] = inv->next; free(inv); inv = c->invocations_ht[i]; } } else { lastinv = inv; inv = inv->next; } } } pthread_rwlock_unlock(&c->invlock); } } if (work == 0) { if (__sync_bool_compare_and_swap(&keep_running, 0, 0)) break; /* nothing done, avoid spinlocking */ usleep(250 * 1000); /* 250ms */ } } /* free up value buckets */ while ((s = aggrs) != NULL) { while (s->computes != NULL) { c = s->computes; free((void *)c->metric); for (i = 0; i < 1 << AGGR_HT_POW_SIZE; i++) { inv = c->invocations_ht[i]; while (inv != NULL) { struct _aggr_invocations *invocation = inv; free(inv->metric); if (c->entries_needed) for (j = 0; j < s->bucketcnt; j++) if (inv->buckets[j].entries.values) free(inv->buckets[j].entries.values); free(inv->buckets); inv = invocation->next; free(invocation); } } pthread_rwlock_destroy(&c->invlock); s->computes = c->next; free(c); } aggrs = aggrs->next; free(s); } return NULL; } /** * Returns the number of aggregators defined. */ size_t aggregator_numaggregators(aggregator *aggrs) { size_t totaggregators = 0; aggregator *a; for (a = aggrs; a != NULL; a = a->next) totaggregators++; return totaggregators; } /** * Returns the total number of computations defined. */ size_t aggregator_numcomputes(aggregator *aggrs) { size_t totcomputes = 0; aggregator *a; struct _aggr_computes *c; for (a = aggrs; a != NULL; a = a->next) for (c = a->computes; c != NULL; c = c->next) totcomputes++; return totcomputes; } /** * Initialises and starts the aggregator. Returns false when starting * failed, true otherwise. */ int aggregator_start(aggregator *aggrs) { keep_running = 1; if (pthread_create(&aggregatorid, NULL, aggregator_expire, aggrs) != 0) return 0; return 1; } /** * Shuts down the aggregator. */ void aggregator_stop(void) { __sync_bool_compare_and_swap(&keep_running, 1, 0); pthread_join(aggregatorid, NULL); } /** * Returns an approximate number of received metrics by all aggregators. */ size_t aggregator_get_received(aggregator *a) { size_t totreceived = 0; for ( ; a != NULL; a = a->next) totreceived += __sync_add_and_fetch(&a->received, 0); return totreceived; } /** * Returns an approximate number of metrics received by all aggregators * since the last call to this function. */ inline size_t aggregator_get_received_sub(aggregator *aggrs) { size_t r = aggregator_get_received(aggrs) - prevreceived; prevreceived += r; return r; } /** * Returns an approximate number of metrics sent by all aggregators. */ size_t aggregator_get_sent(aggregator *a) { size_t totsent = 0; for ( ; a != NULL; a = a->next) totsent += __sync_add_and_fetch(&a->sent, 0); return totsent; } /** * Returns an approximate number of metrics sent by all aggregators * since the last call to this function. */ inline size_t aggregator_get_sent_sub(aggregator *aggrs) { size_t r = aggregator_get_sent(aggrs) - prevsent; prevsent += r; return r; } /** * Returns an approximate number of dropped metrics by all aggregators. * Metrics are dropped if they are too much in the past (past expiry * time) or if they are too much in the future. */ size_t aggregator_get_dropped(aggregator *a) { size_t totdropped = 0; for ( ; a != NULL; a = a->next) totdropped += __sync_add_and_fetch(&a->dropped, 0); return totdropped; } /** * Returns an approximate number of metrics dropped by all aggregators * since the last call to this function. */ inline size_t aggregator_get_dropped_sub(aggregator *aggrs) { size_t r = aggregator_get_dropped(aggrs) - prevdropped; prevdropped += r; return r; } carbon-c-relay-2.5/aggregator.h000066400000000000000000000053721303466152100164610ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef AGGREGATOR_H #define AGGREGATOR_H 1 #include #include #include "server.h" #define AGGR_HT_POW_SIZE 12 /* 4096: too big? issue #60 */ typedef struct _aggregator { unsigned short interval; /* when to perform the aggregation */ unsigned short expire; /* when incoming metrics are no longer valid */ enum _aggr_timestamp { TS_START, TS_MIDDLE, TS_END } tswhen; unsigned char bucketcnt; int disp_conn; int fd; size_t received; size_t sent; size_t dropped; struct _aggr_computes { enum _aggr_compute_type { SUM, CNT, MAX, MIN, AVG, MEDN, PCTL, VAR, SDEV } type; const char *metric; /* name template of metric to produce */ struct _aggr_invocations { char *metric; /* actual name to emit */ unsigned int hash; /* to speed up matching */ unsigned short expire; /* expire + splay */ struct _aggr_bucket { long long int start; size_t cnt; double sum; double max; double min; struct _aggr_bucket_entries { size_t size; double *values; } entries; } *buckets; struct _aggr_invocations *next; } *invocations_ht[1 << AGGR_HT_POW_SIZE]; unsigned char entries_needed:1; unsigned char percentile:7; pthread_rwlock_t invlock; struct _aggr_computes *next; } *computes; struct _aggregator *next; } aggregator; aggregator *aggregator_new(unsigned int interval, unsigned int expire, enum _aggr_timestamp tswhen); char aggregator_add_compute(aggregator *s, const char *metric, const char *type); void aggregator_set_stub(aggregator *s, const char *stubname); void aggregator_putmetric(aggregator *s, const char *metric, const char *firstspace, size_t nmatch, regmatch_t *pmatch); int aggregator_start(aggregator *aggrs); void aggregator_stop(void); size_t aggregator_numaggregators(aggregator *agrs); size_t aggregator_numcomputes(aggregator *aggrs); size_t aggregator_get_received(aggregator *aggrs); size_t aggregator_get_sent(aggregator *aggrs); size_t aggregator_get_dropped(aggregator *aggrs); size_t aggregator_get_received_sub(aggregator *aggrs); size_t aggregator_get_sent_sub(aggregator *aggrs); size_t aggregator_get_dropped_sub(aggregator *aggrs); #endif carbon-c-relay-2.5/carbon-c-relay.1000066400000000000000000001170011303466152100170370ustar00rootroot00000000000000.\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . .TH "CARBON\-C\-RELAY" "1" "January 2017" "Graphite" "Graphite data collection and visualisation" . .SH "NAME" \fBcarbon\-c\-relay\fR \- graphite relay, aggregator and rewriter . .SH "SYNOPSIS" \fBcarbon\-c\-relay\fR \fB\-f\fR \fIconfig\-file\fR \fB[ options\fR \.\.\. \fB]\fR . .SH "DESCRIPTION" \fBcarbon\-c\-relay\fR accepts, cleanses, matches, rewrites, forwards and aggregates graphite metrics by listening for incoming connections and relaying the messages to other servers defined in its configuration\. The core functionality is to route messages via flexible rules to the desired destinations\. . .P \fBcarbon\-c\-relay\fR is a simple program that reads its routing information from a file\. The command line arguments allow to set the location for this file, as well as the amount of dispatchers (worker threads) to use for reading the data from incoming connections and passing them onto the right destination(s)\. The route file supports two main constructs: clusters and matches\. The first define groups of hosts data metrics can be sent to, the latter define which metrics should be sent to which cluster\. Aggregation rules are seen as matches\. . .P For every metric received by the relay, cleansing is performed\. The following changes are performed before any match, aggregate or rewrite rule sees the metric: . .IP "\(bu" 4 double dot elimination (necessary for correctly functioning consistent hash routing) . .IP "\(bu" 4 trailing/leading dot elimination . .IP "\(bu" 4 whitespace normalisation (this mostly affects output of the relay to other targets: metric, value and timestamp will be separated by a single space only, ever) . .IP "\(bu" 4 irregular char replacement with underscores (_), currently irregular is defined as not being in \fB[0\-9a\-zA\-Z\-_:#]\fR, but can be overridden on the command line\. . .IP "" 0 . .SH "OPTIONS" These options control the behaviour of \fBcarbon\-c\-relay\fR\. . .TP \fB\-v\fR Print version string and exit\. . .TP \fB\-d\fR Enable debug mode, this prints statistics to stdout and prints extra messages about some situations encountered by the relay that normally would be too verbose to be enabled\. When combined with \fB\-t\fR (test mode) this also prints stub routes and consistent\-hash ring contents\. . .TP \fB\-m\fR Change statistics submission to be like carbon\-cache\.py, e\.g\. not cumulative\. After each submission, all counters are reset to \fB0\fR\. . .TP \fB\-s\fR Enable submission mode\. In this mode, internal statistics are not generated\. Instead, queue pressure and metrics drops are reported on stdout\. This mode is useful when used as submission relay which\' job is just to forward to (a set of) main relays\. Statistics about the submission relays in this case are not needed, and could easily cause a non\-desired flood of metrics e\.g\. when used on each and every host locally\. . .TP \fB\-t\fR Test mode\. This mode doesn\'t do any routing at all, but instead reads input from stdin and prints what actions would be taken given the loaded configuration\. This mode is very useful for testing relay routes for regular expression syntax etc\. It also allows to give insight on how routing is applied in complex configurations, for it shows rewrites and aggregates taking place as well\. . .TP \fB\-D\fR Deamonise into the background after startup\. This option requires \fB\-l\fR and \fB\-P\fR flags to be set as well\. . .TP \fB\-f\fR \fIconfig\-file\fR Read configuration from \fIconfig\-file\fR\. A configuration consists of clusters and routes\. See \fICONFIGURATION SYNTAX\fR for more information on the options and syntax of this file\. . .TP \fB\-i\fR \fIiface\fR Open up connections on interface \fIiface\fR\. Currently only one interface can be specified, and it is specified by its IP address, not the interface name\. By default, the relay opens listeners on all available interfaces (a\.k\.a\. \fB0\.0\.0\.0\fR)\. . .TP \fB\-l\fR \fIlog\-file\fR Use \fIlog\-file\fR for writing messages\. Without this option, the relay writes both to \fIstdout\fR and \fIstderr\fR\. When logging to file, all messages are prefixed with \fBMSG\fR when they were sent to \fIstdout\fR, and \fBERR\fR when they were sent to \fIstderr\fR\. . .TP \fB\-p\fR \fIport\fR Listen for connections on port \fIport\fR\. The port number is used for both \fBTCP\fR, \fBUDP\fR and \fBUNIX sockets\fR\. In the latter case, the socket file contains the port number\. The port defaults to \fI2003\fR, which is also used by the original \fBcarbon\-cache\.py\fR\. . .TP \fB\-w\fR \fIworkers\fR Use \fIworkers\fR number of threads\. The default number of workers is equal to the amount of detected CPU cores\. It makes sense to reduce this number on many\-core machines, or when the traffic is low\. . .TP \fB\-b\fR \fIbatchsize\fR Set the amount of metrics that sent to remote servers at once to \fIbatchsize\fR\. When the relay sends metrics to servers, it will retrieve \fBbatchsize\fR metrics from the pending queue of metrics waiting for that server and send those one by one\. The size of the batch will have minimal impact on sending performance, but it controls the amount of lock\-contention on the queue\. The default is \fI2500\fR\. . .TP \fB\-q\fR \fIqueuesize\fR Each server from the configuration where the relay will send metrics to, has a queue associated with it\. This queue allows for disruptions and bursts to be handled\. The size of this queue will be set to \fIqueuesize\fR which allows for that amount of metrics to be stored in the queue before it overflows, and the relay starts dropping metrics\. The larger the queue, more metrics can be absorbed, but also more memory will be used by the relay\. The default queue size is \fI25000\fR\. . .TP \fB\-L\fR \fIstalls\fR Sets the max mount of stalls to \fIstalls\fR before the relay starts dropping metrics for a server\. When a queue fills up, the relay uses a mechanism called stalling to signal the client (writing to the relay) of this event\. In particular when the client sends a large amount of metrics in very short time (burst), stalling can help to avoid dropping metrics, since the client just needs to slow down for a bit, which in many cases is possible (e\.g\. when catting a file with \fBnc\fR(1))\. However, this behaviour can also obstruct, artificially stalling writers which cannot stop that easily\. For this the stalls can be set from \fI0\fR to \fI15\fR, where each stall can take around 1 second on the client\. The default value is set to \fI4\fR, which is aimed at the occasional disruption scenario and max effort to not loose metrics with moderate slowing down of clients\. . .TP \fB\-S\fR \fIinterval\fR Set the interval in which statistics are being generated and sent by the relay to \fIinterval\fR seconds\. The default is \fI60\fR\. . .TP \fB\-B\fR \fIbacklog\fR Sets TCP connection listen backlog to \fIbacklog\fR connections\. The default value is \fI3\fR but on servers which receive many concurrent connections, this setting likely needs to be increased to avoid connection refused errors on the clients\. . .TP \fB\-U\fR \fIbufsize\fR Sets the socket send/receive buffer sizes in bytes\. When unset, the OS default is used\. The maximum is also determined by the OS\. The sizes are set using setsockopt with the flags SO_RCVBUF and SO_SNDBUF\. Setting this size may be necessary for large volume scenarios, for which also \fB\-B\fR might apply\. . .TP \fB\-T\fR \fItimeout\fR Specifies the IO timeout in milliseconds used for server connections\. The default is \fI600\fR milliseconds, but may need increasing when WAN links are used for target servers\. A relatively low value for connection timeout allows the relay to quickly establish a server is unreachable, and as such failover strategies to kick in before the queue runs high\. . .TP \fB\-c\fR \fIchars\fR Defines the characters that are next to \fB[A\-Za\-z0\-9]\fR allowed in metrics to \fIchars\fR\. Any character not in this list, is replaced by the relay with \fB_\fR (underscore)\. The default list of allowed characters is \fI\-_:#\fR\. . .TP \fB\-H\fR \fIhostname\fR Override hostname determined by a call to \fBgethostname\fR(3) with \fIhostname\fR\. The hostname is used mainly in the statistics metrics \fBcarbon\.relays\.\.<\.\.\.>\fR sent by the relay\. . .TP \fB\-P\fR \fIpidfile\fR Write the pid of the relay process to a file called \fIpidfile\fR\. This is in particular useful when daemonised in combination with init managers\. . .SH "CONFIGURATION SYNTAX" The config file supports the following syntax, where comments start with a \fB#\fR character and can appear at any position on a line and suppress input until the end of that line: . .IP "" 4 . .nf cluster [replication ]> ]> \.\.\. ; cluster file [ip] \.\.\. ; match <* | expression \.\.\.> [validate else ] send to [stop] ; rewrite into ; aggregate \.\.\. every seconds expire after seconds [timestamp at of bucket] compute | variance | stddev> write to [compute \.\.\.] [send to ] [stop] ; send statistics to [stop] ; include ; . .fi . .IP "" 0 . .P Multiple clusters can be defined, and need not to be referenced by a match rule\. All clusters point to one or more hosts, except the \fBfile\fR cluster which writes to files in the local filesystem\. \fBhost\fR may be an IPv4 or IPv6 address, or a hostname\. Since host is followed by an optional \fB:\fR and port, for IPv6 addresses not to be interpreted wrongly, either a port must be given, or the IPv6 address surrounded by brackets, e\.g\. \fB[::1]\fR\. An optional \fBproto udp\fR or \fBproto tcp\fR may be added to specify the use of UDP or TCP to connect to the remote server\. When omitted this defaults to a TCP connection\. . .P The \fBforward\fR and \fBfile\fR clusters simply send everything they receive to all defined members (host addresses or files)\. The \fBany_of\fR cluster is a small variant of the \fBforward\fR cluster, but instead of sending to all defined members, it sends each incoming metric to one of defined members\. This is not much useful in itself, but since any of the members can receive each metric, this means that when one of the members is unreachable, the other members will receive all of the metrics\. This can be useful when the cluster points to other relays\. The \fBany_of\fR router tries to send the same metrics consistently to the same destination\. The \fBfailover\fR cluster is like the \fBany_of\fR cluster, but sticks to the order in which servers are defined\. This is to implement a pure failover scenario between servers\. The \fBcarbon_ch\fR cluster sends the metrics to the member that is responsible according to the consistent hash algorithm (as used in the original carbon), or multiple members if replication is set to more than 1\. The \fBfnv1a_ch\fR cluster is a identical in behaviour to \fBcarbon_ch\fR, but it uses a different hash technique (FNV1a) which is faster but more importantly defined to get by a limitation of \fBcarbon_ch\fR to use both host and port from the members\. This is useful when multiple targets live on the same host just separated by port\. The instance that original carbon uses to get around this can be set by appending it after the port, separated by an equals sign, e\.g\. \fB127\.0\.0\.1:2006=a\fR for instance \fBa\fR\. When using the \fBfnv1a_ch\fR cluster, this instance overrides the hash key in use\. This allows for many things, including masquerading old IP addresses, but mostly to make the hash key location to become agnostic of the (physical) location of that key\. For example, usage like \fB10\.0\.0\.1:2003=4d79d13554fa1301476c1f9fe968b0ac\fR would allow to change port and/or ip address of the server that receives data for the instance key\. Obviously, this way migration of data can be dealt with much more conveniently\. The \fBjump_fnv1a_ch\fR cluster is also a consistent hash cluster like the previous two, but it does not take the server information into account at all\. Whether this is useful to you depends on your scenario\. The jump hash has a much better balancing over the servers defined in the cluster, at the expense of not being able to remove any server but the last in order\. What this means is that this hash is fine to use with ever growing clusters where older nodes are also replaced at some point\. If you have a cluster where removal of old nodes takes place often, the jump hash is not suitable for you\. Jump hash works with servers in an ordered list without gaps\. To influence the ordering, the instance given to the server will be used as sorting key\. Without, the order will be as given in the file\. It is a good practice to fix the order of the servers with instances such that it is explicit what the right nodes for the jump hash are\. . .P DNS hostnames are resolved to a single address, according to the preference rules in RFC 3484 \fIhttps://www\.ietf\.org/rfc/rfc3484\.txt\fR\. The \fBany_of\fR cluster has an explicit \fBuseall\fR flag that enables a hostname to resolve to multiple addresses\. Each address returned becomes a cluster destination\. . .P Match rules are the way to direct incoming metrics to one or more clusters\. Match rules are processed top to bottom as they are defined in the file\. It is possible to define multiple matches in the same rule\. Each match rule can send data to one or more clusters\. Since match rules "fall through" unless the \fBstop\fR keyword is added, carefully crafted match expression can be used to target multiple clusters or aggregations\. This ability allows to replicate metrics, as well as send certain metrics to alternative clusters with careful ordering and usage of the \fBstop\fR keyword\. The special cluster \fBblackhole\fR discards any metrics sent to it\. This can be useful for weeding out unwanted metrics in certain cases\. Because throwing metrics away is pointless if other matches would accept the same data, a match with as destination the blackhole cluster, has an implicit \fBstop\fR\. The \fBvalidation\fR clause adds a check to the data (what comes after the metric) in the form of a regular expression\. When this expression matches, the match rule will execute as if no validation clause was present\. However, if it fails, the match rule is aborted, and no metrics will be sent to destinations, this is the \fBdrop\fR behaviour\. When \fBlog\fR is used, the metric is logged to stderr\. Care should be taken with the latter to avoid log flooding\. When a validate clause is present, destinations need not to be present, this allows for applying a global validation rule\. Note that the cleansing rules are applied before validation is done, thus the data will not have duplicate spaces\. . .P Rewrite rules take a regular input to match incoming metrics, and transform them into the desired new metric name\. In the replacement, backreferences are allowed to match capture groups defined in the input regular expression\. A match of \fBserver\e\.(x|y|z)\e\.\fR allows to use e\.g\. \fBrole\.\e1\.\fR in the substitution\. A few caveats apply to the current implementation of rewrite rules\. First, their location in the config file determines when the rewrite is performed\. The rewrite is done in\-place, as such a match rule before the rewrite would match the original name, a match rule after the rewrite no longer matches the original name\. Care should be taken with the ordering, as multiple rewrite rules in succession can take place, e\.g\. \fBa\fR gets replaced by \fBb\fR and \fBb\fR gets replaced by \fBc\fR in a succeeding rewrite rule\. The second caveat with the current implementation, is that the rewritten metric names are not cleansed, like newly incoming metrics are\. Thus, double dots and potential dangerous characters can appear if the replacement string is crafted to produce them\. It is the responsibility of the writer to make sure the metrics are clean\. If this is an issue for routing, one can consider to have a rewrite\-only instance that forwards all metrics to another instance that will do the routing\. Obviously the second instance will cleanse the metrics as they come in\. The backreference notation allows to lowercase and uppercase the replacement string with the use of the underscore (\fB_\fR) and carret (\fB^\fR) symbols following directly after the backslash\. For example, \fBrole\.\e_1\.\fR as substitution will lowercase the contents of \fB\e1\fR\. . .P The aggregations defined take one or more input metrics expressed by one or more regular expresions, similar to the match rules\. Incoming metrics are aggregated over a period of time defined by the interval in seconds\. Since events may arrive a bit later in time, the expiration time in seconds defines when the aggregations should be considered final, as no new entries are allowed to be added any more\. On top of an aggregation multiple aggregations can be computed\. They can be of the same or different aggregation types, but should write to a unique new metric\. The metric names can include back references like in rewrite expressions, allowing for powerful single aggregation rules that yield in many aggregations\. When no \fBsend to\fR clause is given, produced metrics are sent to the relay as if they were submitted from the outside, hence match and aggregation rules apply to those\. Care should be taken that loops are avoided this way\. For this reason, the use of the \fBsend to\fR clause is encouraged, to direct the output traffic where possible\. Like for match rules, it is possible to define multiple cluster targets\. Also, like match rules, the \fBstop\fR keyword applies to control the flow of metrics in the matching process\. . .P The special \fBsend statistics to\fR construct is much like a \fBmatch\fR rule which matches the (internal) statistics produced by the relay\. It can be used to avoid router loops when sending the statistics to a certain destination\. The \fBsend statistics\fR construct can only be used once, but multiple destinations can be used then required\. . .P In case configuration becomes very long, or is managed better in separate files, the \fBinclude\fR directive can be used to read another file\. The given file will be read in place and added to the router configuration at the time of inclusion\. The end result is one big route configuration\. Multiple \fBinclude\fR statements can be used throughout the configuration file\. The positioning will influence the order of rules as normal\. Beware that recursive inclusion (\fBinclude\fR from an included file) is supported, and currently no safeguards exist for an inclusion loop\. For what is worth, this feature likely is best used with simple configuration files (e\.g\. not having \fBinclude\fR in them)\. . .SH "EXAMPLES" \fBcarbon\-c\-relay\fR evolved over time, growing features on demand as the tool proved to be stable and fitting the job well\. Below follow some annotated examples of constructs that can be used with the relay\. . .P Clusters can be defined as much as necessary\. They receive data from match rules, and their type defines which members of the cluster finally get the metric data\. The simplest cluster form is a \fBforward\fR cluster: . .IP "" 4 . .nf cluster send\-through forward 10\.1\.0\.1 ; . .fi . .IP "" 0 . .P Any metric sent to the \fBsend\-through\fR cluster would simply be forwarded to the server at IPv4 address \fB10\.1\.0\.1\fR\. If we define multiple servers, all of those servers would get the same metric, thus: . .IP "" 4 . .nf cluster send\-through forward 10\.1\.0\.1 10\.2\.0\.1 ; . .fi . .IP "" 0 . .P The above results in a duplication of metrics send to both machines\. This can be useful, but most of the time it is not\. The \fBany_of\fR cluster type is like \fBforward\fR, but it sends each incoming metric to any of the members\. The same example with such cluster would be: . .IP "" 4 . .nf cluster send\-to\-any\-one any_of 10\.1\.0\.1:2010 10\.1\.0\.1:2011; . .fi . .IP "" 0 . .P This would implement a multipath scenario, where two servers are used, the load between them is spread, but should any of them fail, all metrics are sent to the remaining one\. This typically works well for upstream relays, or for balancing carbon\-cache processes running on the same machine\. Should any member become unavailable, for instance due to a rolling restart, the other members receive the traffic\. If it is necessary to have true fail\-over, where the secondary server is only used if the first is down, the following would implement that: . .IP "" 4 . .nf cluster try\-first\-then\-second failover 10\.1\.0\.1:2010 10\.1\.0\.1:2011; . .fi . .IP "" 0 . .P These types are different from the two consistent hash cluster types: . .IP "" 4 . .nf cluster graphite carbon_ch 127\.0\.0\.1:2006=a 127\.0\.0\.1:2007=b 127\.0\.0\.1:2008=c ; . .fi . .IP "" 0 . .P If a member in this example fails, all metrics that would go to that member are kept in the queue, waiting for the member to return\. This is useful for clusters of carbon\-cache machines where it is desirable that the same metric ends up on the same server always\. The \fBcarbon_ch\fR cluster type is compatible with carbon\-relay consistent hash, and can be used for existing clusters populated by carbon\-relay\. For new clusters, however, it is better to use the \fBfnv1a_ch\fR cluster type, for it is faster, and allows to balance over the same address but different ports without an instance number, in constrast to \fBcarbon_ch\fR\. . .P Because we can use multiple clusters, we can also replicate without the use of the \fBforward\fR cluster type, in a more intelligent way: . .IP "" 4 . .nf cluster dc\-old carbon_ch replication 2 10\.1\.0\.1 10\.1\.0\.2 10\.1\.0\.3 ; cluster dc\-new1 fnv1a_ch replication 2 10\.2\.0\.1 10\.2\.0\.2 10\.2\.0\.3 ; cluster dc\-new2 fnv1a_ch replication 2 10\.3\.0\.1 10\.3\.0\.2 10\.3\.0\.3 ; match * send to dc\-old ; match * send to dc\-new1 dc\-new2 stop ; . .fi . .IP "" 0 . .P In this example all incoming metrics are first sent to \fBdc\-old\fR, then \fBdc\-new1\fR and finally to \fBdc\-new2\fR\. Note that the cluster type of \fBdc\-old\fR is different\. Each incoming metric will be send to 2 members of all three clusters, thus replicating to in total 6 destinations\. For each cluster the destination members are computed independently\. Failure of clusters or members does not affect the others, since all have individual queues\. The above example could also be written using three match rules for each dc, or one match rule for all three dcs\. The difference is mainly in performance, the number of times the incoming metric has to be matched against an expression\. The \fBstop\fR rule in \fBdc\-new\fR match rule is not strictly necessary in this example, because there are no more following match rules\. However, if the match would target a specific subset, e\.g\. \fB^sys\e\.\fR, and more clusters would be defined, this could be necessary, as for instance in the following abbreviated example: . .IP "" 4 . .nf cluster dc1\-sys \.\.\. ; cluster dc2\-sys \.\.\. ; cluster dc1\-misc \.\.\. ; cluster dc2\-misc \.\.\. ; match ^sys\e\. send to dc1\-sys; match ^sys\e\. send to dc2\-sys stop; match * send to dc1\-misc; match * send to dc2\-misc stop; . .fi . .IP "" 0 . .P As can be seen, without the \fBstop\fR in dc2\-sys\' match rule, all metrics starting with \fBsys\.\fR would also be send to dc1\-misc and dc2\-misc\. It can be that this is desired, of course, but in this example there is a dedicated cluster for the \fBsys\fR metrics\. . .P Suppose there would be some unwanted metric that unfortunately is generated, let\'s assume some bad/old software\. We don\'t want to store this metric\. The \fBblackhole\fR cluster is suitable for that, when it is harder to actually whitelist all wanted metrics\. Consider the following: . .IP "" 4 . .nf match some_legacy1$ some_legacy2$ send to blackhole stop; . .fi . .IP "" 0 . .P This would throw away all metrics that end with \fBsome_legacy\fR, that would otherwise be hard to filter out\. Since the order matters, it can be used in a construct like this: . .IP "" 4 . .nf cluster old \.\.\. ; cluster new \.\.\. ; match * send to old; match unwanted send to blackhole stop; match * send to new; . .fi . .IP "" 0 . .P In this example the old cluster would receive the metric that\'s unwanted for the new cluster\. So, the order in which the rules occur does matter for the execution\. . .P Validation can be used to ensure the data for metrics is as expected\. A global validation for just number (no floating point) values could be: . .IP "" 4 . .nf match * validate ^[0\-9]+\e [0\-9]+$ else drop ; . .fi . .IP "" 0 . .P (Note the escape with backslash \fB\e\fR of the space, you might be able to use \fB\es\fR or \fB[:space:]\fR instead, this depends on your libc implementation\.) . .P The validation clause can exist on every match rule, so in principle, the following is valid: . .IP "" 4 . .nf match ^foo validate ^[0\-9]+\e [0\-9]+$ else drop send to integer\-cluster ; match ^foo validate ^[0\-9\.e+\-]+\e [0\-9\.e+\-]+$ else drop send to float\-cluster stop; . .fi . .IP "" 0 . .P Note that the behaviour is different in the previous two examples\. When no \fBsend to\fR clusters are specified, a validation error makes the match behave like the \fBstop\fR keyword is present\. Likewise, when validation passes, processing continues with the next rule\. When destination clusters are present, the \fBmatch\fR respects the \fBstop\fR keyword as normal\. When specified, processing will always stop when specified so\. However, if validation fails, the rule does not send anything to the destination clusters, the metric will be dropped or logged, but never sent\. . .P The relay is capable of rewriting incoming metrics on the fly\. This process is done based on regular expressions with capture groups that allow to substitute parts in a replacement string\. Rewrite rules allow to cleanup metrics from applications, or provide a migration path\. In it\'s simplest form a rewrite rule looks like this: . .IP "" 4 . .nf rewrite ^server\e\.(\.+)\e\.(\.+)\e\.([a\-zA\-Z]+)([0\-9]+) into server\.\e_1\.\e2\.\e3\.\e3\e4 ; . .fi . .IP "" 0 . .P In this example a metric like \fBserver\.DC\.role\.name123\fR would be transformed into \fBserver\.dc\.role\.name\.name123\fR\. For rewrite rules hold the same as for matches, that their order matters\. Hence to build on top of the old/new cluster example done earlier, the following would store the original metric name in the old cluster, and the new metric name in the new cluster: . .IP "" 4 . .nf match * send to old; rewrite \.\.\. ; match * send to new; . .fi . .IP "" 0 . .P Note that after the rewrite, the original metric name is no longer available, as the rewrite happens in\-place\. . .P Aggregations are probably the most complex part of carbon\-c\-relay\. Two ways of specifying aggregates are supported by carbon\-c\-relay\. The first, static rules, are handled by an optimiser which tries to fold thousands of rules into groups to make the matching more efficient\. The second, dynamic rules, are very powerful compact definitions with possibly thousands of internal instantiations\. A typical static aggregation looks like: . .IP "" 4 . .nf aggregate ^sys\e\.dc1\e\.somehost\-[0\-9]+\e\.somecluster\e\.mysql\e\.replication_delay ^sys\e\.dc2\e\.somehost\-[0\-9]+\e\.somecluster\e\.mysql\e\.replication_delay every 10 seconds expire after 35 seconds timestamp at end of bucket compute sum write to mysql\.somecluster\.total_replication_delay compute average write to mysql\.somecluster\.average_replication_delay compute max write to mysql\.somecluster\.max_replication_delay compute count write to mysql\.somecluster\.replication_delay_metric_count ; . .fi . .IP "" 0 . .P In this example, four aggregations are produced from the incoming matching metrics\. In this example we could have written the two matches as one, but for demonstration purposes we did not\. Obviously they can refer to different metrics, if that makes sense\. The \fBevery 10 seconds\fR clause specifies in what interval the aggregator can expect new metrics to arrive\. This interval is used to produce the aggregations, thus each 10 seconds 4 new metrics are generated from the data received sofar\. Because data may be in transit for some reason, or generation stalled, the \fBexpire after\fR clause specifies how long the data should be kept before considering a data bucket (which is aggregated) to be complete\. In the example, 35 was used, which means after 35 seconds the first aggregates are produced\. It also means that metrics can arrive 35 seconds late, and still be taken into account\. The exact time at which the aggregate metrics are produced is random between 0 and interval (10 in this case) seconds after the expiry time\. This is done to prevent thundering herds of metrics for large aggregation sets\. The \fBtimestamp\fR that is used for the aggregations can be specified to be the \fBstart\fR, \fBmiddle\fR or \fBend\fR of the bucket\. Original carbon\-aggregator\.py uses \fBstart\fR, while carbon\-c\-relay\'s default has always been \fBend\fR\. The \fBcompute\fR clauses demonstrate a single aggregation rule can produce multiple aggregates, as often is the case\. Internally, this comes for free, since all possible aggregates are always calculated, whether or not they are used\. The produced new metrics are resubmitted to the relay, hence matches defined before in the configuration can match output of the aggregator\. It is important to avoid loops, that can be generated this way\. In general, splitting aggregations to their own carbon\-c\-relay instance, such that it is easy to forward the produced metrics to another relay instance is a good practice\. . .P The previous example could also be written as follows to be dynamic: . .IP "" 4 . .nf aggregate ^sys\e\.dc[0\-9]\.(somehost\-[0\-9]+)\e\.([^\.]+)\e\.mysql\e\.replication_delay every 10 seconds expire after 35 seconds compute sum write to mysql\.host\.\e1\.replication_delay compute sum write to mysql\.host\.all\.replication_delay compute sum write to mysql\.cluster\.\e2\.replication_delay compute sum write to mysql\.cluster\.all\.replication_delay ; . .fi . .IP "" 0 . .P Here a single match, results in four aggregations, each of a different scope\. In this example aggregation based on hostname and cluster are being made, as well as the more general \fBall\fR targets, which in this example have both identical values\. Note that with this single aggregation rule, both per\-cluster, per\-host and total aggregations are produced\. Obviously, the input metrics define which hosts and clusters are produced\. . .P With use of the \fBsend to\fR clause, aggregations can be made more intuitive and less error\-prone\. Consider the below example: . .IP "" 4 . .nf cluster graphite fnv1a_ch ip1 ip2 ip3; aggregate ^sys\e\.somemetric every 60 seconds expire after 75 seconds compute sum write to sys\.somemetric send to graphite stop ; match * send to graphite; . .fi . .IP "" 0 . .P It sends all incoming metrics to the graphite cluster, except the sys\.somemetric ones, which it replaces with a sum of all the incoming ones\. Without a \fBstop\fR in the aggregate, this causes a loop, and without the \fBsend to\fR, the metric name can\'t be kept its original name, for the output now directly goes to the cluster\. . .SH "STATISTICS" When \fBcarbon\-c\-relay\fR is run without \fB\-d\fR or \fB\-s\fR arguments, statistics will be produced and sent to the relay itself in the form of \fBcarbon\.relays\.\.*\fR\. The hostname is determined on startup, and can be overriden using the \fB\-H\fR argument\. While many metrics have a similar name to what carbon\-cache\.py would produce, their values are different\. To obtain a more compatible set of values, the \fB\-m\fR argument can be used to make values non\-cumulative, that is, they will report the change compared to the previous value\. By default, most values are running counters which only increase over time\. The use of the nonNegativeDerivative() function from graphite is useful with these\. The default sending interval is 1 minute (60 seconds), but can be overridden using the \fB\-S\fR argument specified in seconds\. . .P The following metrics are produced in the \fBcarbon\.relays\.\fR namespace: . .IP "\(bu" 4 metricsReceived . .IP The number of metrics that were received by the relay\. Received here means that they were seen and processed by any of the dispatchers\. . .IP "\(bu" 4 metricsSent . .IP The number of metrics that were sent from the relay\. This is a total count for all servers combined\. When incoming metrics are duplicated by the cluster configuration, this counter will include all those duplications\. In other words, the amount of metrics that were successfully sent to other systems\. Note that metrics that are processed (received) but still in the sending queue (queued) are not included in this counter\. . .IP "\(bu" 4 metricsQueued . .IP The total number of metrics that are currently in the queues for all the server targets\. This metric is not cumulative, for it is a sample of the queue size, which can (and should) go up and down\. Therefore you should not use the derivative function for this metric\. . .IP "\(bu" 4 metricsDropped . .IP The total number of metric that had to be dropped due to server queues overflowing\. A queue typically overflows when the server it tries to send its metrics to is not reachable, or too slow in ingesting the amount of metrics queued\. This can be network or resource related, and also greatly depends on the rate of metrics being sent to the particular server\. . .IP "\(bu" 4 metricsBlackholed . .IP The number of metrics that did not match any rule, or matched a rule with blackhole as target\. Depending on your configuration, a high value might be an indication of a misconfiguration somewhere\. These metrics were received by the relay, but never sent anywhere, thus they disappeared\. . .IP "\(bu" 4 metricStalls . .IP The number of times the relay had to stall a client to indicate that the downstream server cannot handle the stream of metrics\. A stall is only performed when the queue is full and the server is actually receptive of metrics, but just too slow at the moment\. Stalls typically happen during micro\-bursts, where the client typically is unaware that it should stop sending more data, while it is able to\. . .IP "\(bu" 4 connections . .IP The number of connect requests handled\. This is an ever increasing number just counting how many connections were accepted\. . .IP "\(bu" 4 disconnects . .IP The number of disconnected clients\. A disconnect either happens because the client goes away, or due to an idle timeout in the relay\. The difference between this metric and connections is the amount of connections actively held by the relay\. In normal situations this amount remains within reasonable bounds\. Many connections, but few disconnections typically indicate a possible connection leak in the client\. The idle connections disconnect in the relay here is to guard against resource drain in such scenarios\. . .IP "\(bu" 4 dispatch_wallTime_us . .IP The number of microseconds spent by the dispatchers to do their work\. In particular on multi\-core systems, this value can be confusing, however, it indicates how long the dispatchers were doing work handling clients\. It includes everything they do, from reading data from a socket, cleaning up the input metric, to adding the metric to the appropriate queues\. The larger the configuration, and more complex in terms of matches, the more time the dispatchers will spend on the cpu\. But also time they do /not/ spend on the cpu is included in this number\. It is the pure wallclock time the dispatcher was serving a client\. . .IP "\(bu" 4 dispatch_sleepTime_us . .IP The number of microseconds spent by the dispatchers sleeping waiting for work\. When this value gets small (or even zero) the dispatcher has so much work that it doesn\'t sleep any more, and likely can\'t process the work in a timely fashion any more\. This value plus the wallTime from above sort of sums up to the total uptime taken by this dispatcher\. Therefore, expressing the wallTime as percentage of this sum gives the busyness percentage draining all the way up to 100% if sleepTime goes to 0\. . .IP "\(bu" 4 server_wallTime_us . .IP The number of microseconds spent by the servers to send the metrics from their queues\. This value includes connection creation, reading from the queue, and sending metrics over the network\. . .IP "\(bu" 4 dispatcherX . .IP For each indivual dispatcher, the metrics received and blackholed plus the wall clock time\. The values are as described above\. . .IP "\(bu" 4 destinations\.X . .IP For all known destinations, the number of dropped, queued and sent metrics plus the wall clock time spent\. The values are as described above\. . .IP "\(bu" 4 aggregators\.metricsReceived . .IP The number of metrics that were matched an aggregator rule and were accepted by the aggregator\. When a metric matches multiple aggregators, this value will reflect that\. A metric is not counted when it is considered syntactically invalid, e\.g\. no value was found\. . .IP "\(bu" 4 aggregators\.metricsDropped . .IP The number of metrics that were sent to an aggregator, but did not fit timewise\. This is either because the metric was too far in the past or future\. The expire after clause in aggregate statements controls how long in the past metric values are accepted\. . .IP "\(bu" 4 aggregators\.metricsSent . .IP The number of metrics that were sent from the aggregators\. These metrics were produced and are the actual results of aggregations\. . .IP "" 0 . .SH "BUGS" Please report them at: \fIhttps://github\.com/grobian/carbon\-c\-relay/issues\fR . .SH "AUTHOR" Fabian Groffen . .SH "SEE ALSO" All other utilities from the graphite stack\. . .P This project aims to be a fast replacement of the original Carbon relay \fIhttp://graphite\.readthedocs\.org/en/1\.0/carbon\-daemons\.html#carbon\-relay\-py\fR\. \fBcarbon\-c\-relay\fR aims to deliver performance and configurability\. Carbon is single threaded, and sending metrics to multiple consistent\-hash clusters requires chaining of relays\. This project provides a multithreaded relay which can address multiple targets and clusters for each and every metric based on pattern matches\. . .P There are a couple more replacement projects out there, which are carbon\-relay\-ng \fIhttps://github\.com/graphite\-ng/carbon\-relay\-ng\fR and graphite\-relay \fIhttps://github\.com/markchadwick/graphite\-relay\fR\. . .P Compared to carbon\-relay\-ng, this project does provide carbon\'s consistent\-hash routing\. graphite\-relay, which does this, however doesn\'t do metric\-based matches to direct the traffic, which this project does as well\. To date, carbon\-c\-relay can do aggregations, failover targets and more\. . .SH "ACKNOWLEDGEMENTS" This program was originally developed for Booking\.com\. With approval from Booking\.com, the code was generalised and published as Open Source on GitHub, for which the author would like to express his gratitude\. Author no longer works for Booking\.com\. carbon-c-relay-2.5/carbon-c-relay.md000066400000000000000000001134361303466152100173070ustar00rootroot00000000000000carbon-c-relay(1) -- graphite relay, aggregator and rewriter ============================================================ [![Build Status](https://travis-ci.org/grobian/carbon-c-relay.svg)](https://travis-ci.org/grobian/carbon-c-relay) ## SYNOPSIS `carbon-c-relay` `-f` *config-file* `[ options` ... `]` ## DESCRIPTION **carbon-c-relay** accepts, cleanses, matches, rewrites, forwards and aggregates graphite metrics by listening for incoming connections and relaying the messages to other servers defined in its configuration. The core functionality is to route messages via flexible rules to the desired destinations. **carbon-c-relay** is a simple program that reads its routing information from a file. The command line arguments allow to set the location for this file, as well as the amount of dispatchers (worker threads) to use for reading the data from incoming connections and passing them onto the right destination(s). The route file supports two main constructs: clusters and matches. The first define groups of hosts data metrics can be sent to, the latter define which metrics should be sent to which cluster. Aggregation rules are seen as matches. For every metric received by the relay, cleansing is performed. The following changes are performed before any match, aggregate or rewrite rule sees the metric: - double dot elimination (necessary for correctly functioning consistent hash routing) - trailing/leading dot elimination - whitespace normalisation (this mostly affects output of the relay to other targets: metric, value and timestamp will be separated by a single space only, ever) - irregular char replacement with underscores (\_), currently irregular is defined as not being in `[0-9a-zA-Z-_:#]`, but can be overridden on the command line. ## OPTIONS These options control the behaviour of **carbon-c-relay**. * `-v`: Print version string and exit. * `-d`: Enable debug mode, this prints statistics to stdout and prints extra messages about some situations encountered by the relay that normally would be too verbose to be enabled. When combined with `-t` (test mode) this also prints stub routes and consistent-hash ring contents. * `-m`: Change statistics submission to be like carbon-cache.py, e.g. not cumulative. After each submission, all counters are reset to `0`. * `-s`: Enable submission mode. In this mode, internal statistics are not generated. Instead, queue pressure and metrics drops are reported on stdout. This mode is useful when used as submission relay which' job is just to forward to (a set of) main relays. Statistics about the submission relays in this case are not needed, and could easily cause a non-desired flood of metrics e.g. when used on each and every host locally. * `-t`: Test mode. This mode doesn't do any routing at all, but instead reads input from stdin and prints what actions would be taken given the loaded configuration. This mode is very useful for testing relay routes for regular expression syntax etc. It also allows to give insight on how routing is applied in complex configurations, for it shows rewrites and aggregates taking place as well. * `-D`: Deamonise into the background after startup. This option requires `-l` and `-P` flags to be set as well. * `-f` *config-file*: Read configuration from *config-file*. A configuration consists of clusters and routes. See [CONFIGURATION SYNTAX][] for more information on the options and syntax of this file. * `-i` *iface*: Open up connections on interface *iface*. Currently only one interface can be specified, and it is specified by its IP address, not the interface name. By default, the relay opens listeners on all available interfaces (a.k.a. `0.0.0.0`). * `-l` *log-file*: Use *log-file* for writing messages. Without this option, the relay writes both to *stdout* and *stderr*. When logging to file, all messages are prefixed with `MSG` when they were sent to *stdout*, and `ERR` when they were sent to *stderr*. * `-p` *port*: Listen for connections on port *port*. The port number is used for both `TCP`, `UDP` and `UNIX sockets`. In the latter case, the socket file contains the port number. The port defaults to *2003*, which is also used by the original `carbon-cache.py`. * `-w` *workers*: Use *workers* number of threads. The default number of workers is equal to the amount of detected CPU cores. It makes sense to reduce this number on many-core machines, or when the traffic is low. * `-b` *batchsize*: Set the amount of metrics that sent to remote servers at once to *batchsize*. When the relay sends metrics to servers, it will retrieve `batchsize` metrics from the pending queue of metrics waiting for that server and send those one by one. The size of the batch will have minimal impact on sending performance, but it controls the amount of lock-contention on the queue. The default is *2500*. * `-q` *queuesize*: Each server from the configuration where the relay will send metrics to, has a queue associated with it. This queue allows for disruptions and bursts to be handled. The size of this queue will be set to *queuesize* which allows for that amount of metrics to be stored in the queue before it overflows, and the relay starts dropping metrics. The larger the queue, more metrics can be absorbed, but also more memory will be used by the relay. The default queue size is *25000*. * `-L` *stalls*: Sets the max mount of stalls to *stalls* before the relay starts dropping metrics for a server. When a queue fills up, the relay uses a mechanism called stalling to signal the client (writing to the relay) of this event. In particular when the client sends a large amount of metrics in very short time (burst), stalling can help to avoid dropping metrics, since the client just needs to slow down for a bit, which in many cases is possible (e.g. when catting a file with `nc`(1)). However, this behaviour can also obstruct, artificially stalling writers which cannot stop that easily. For this the stalls can be set from *0* to *15*, where each stall can take around 1 second on the client. The default value is set to *4*, which is aimed at the occasional disruption scenario and max effort to not loose metrics with moderate slowing down of clients. * `-S` *interval*: Set the interval in which statistics are being generated and sent by the relay to *interval* seconds. The default is *60*. * `-B` *backlog*: Sets TCP connection listen backlog to *backlog* connections. The default value is *3* but on servers which receive many concurrent connections, this setting likely needs to be increased to avoid connection refused errors on the clients. * `-U` *bufsize*: Sets the socket send/receive buffer sizes in bytes. When unset, the OS default is used. The maximum is also determined by the OS. The sizes are set using setsockopt with the flags SO_RCVBUF and SO_SNDBUF. Setting this size may be necessary for large volume scenarios, for which also `-B` might apply. * `-T` *timeout*: Specifies the IO timeout in milliseconds used for server connections. The default is *600* milliseconds, but may need increasing when WAN links are used for target servers. A relatively low value for connection timeout allows the relay to quickly establish a server is unreachable, and as such failover strategies to kick in before the queue runs high. * `-c` *chars*: Defines the characters that are next to `[A-Za-z0-9]` allowed in metrics to *chars*. Any character not in this list, is replaced by the relay with `_` (underscore). The default list of allowed characters is *-_:#*. * `-H` *hostname*: Override hostname determined by a call to `gethostname`(3) with *hostname*. The hostname is used mainly in the statistics metrics `carbon.relays..<...>` sent by the relay. * `-P` *pidfile*: Write the pid of the relay process to a file called *pidfile*. This is in particular useful when daemonised in combination with init managers. ## CONFIGURATION SYNTAX The config file supports the following syntax, where comments start with a `#` character and can appear at any position on a line and suppress input until the end of that line: ``` cluster [replication ]> ]> ... ; cluster file [ip] ... ; match <* | expression ...> [validate else ] send to [stop] ; rewrite into ; aggregate ... every seconds expire after seconds [timestamp at of bucket] compute | variance | stddev> write to [compute ...] [send to ] [stop] ; send statistics to [stop] ; include ; ``` Multiple clusters can be defined, and need not to be referenced by a match rule. All clusters point to one or more hosts, except the `file` cluster which writes to files in the local filesystem. `host` may be an IPv4 or IPv6 address, or a hostname. Since host is followed by an optional `:` and port, for IPv6 addresses not to be interpreted wrongly, either a port must be given, or the IPv6 address surrounded by brackets, e.g. `[::1]`. An optional `proto udp` or `proto tcp` may be added to specify the use of UDP or TCP to connect to the remote server. When omitted this defaults to a TCP connection. The `forward` and `file` clusters simply send everything they receive to all defined members (host addresses or files). The `any_of` cluster is a small variant of the `forward` cluster, but instead of sending to all defined members, it sends each incoming metric to one of defined members. This is not much useful in itself, but since any of the members can receive each metric, this means that when one of the members is unreachable, the other members will receive all of the metrics. This can be useful when the cluster points to other relays. The `any_of` router tries to send the same metrics consistently to the same destination. The `failover` cluster is like the `any_of` cluster, but sticks to the order in which servers are defined. This is to implement a pure failover scenario between servers. The `carbon_ch` cluster sends the metrics to the member that is responsible according to the consistent hash algorithm (as used in the original carbon), or multiple members if replication is set to more than 1. The `fnv1a_ch` cluster is a identical in behaviour to `carbon_ch`, but it uses a different hash technique (FNV1a) which is faster but more importantly defined to get by a limitation of `carbon_ch` to use both host and port from the members. This is useful when multiple targets live on the same host just separated by port. The instance that original carbon uses to get around this can be set by appending it after the port, separated by an equals sign, e.g. `127.0.0.1:2006=a` for instance `a`. When using the `fnv1a_ch` cluster, this instance overrides the hash key in use. This allows for many things, including masquerading old IP addresses, but mostly to make the hash key location to become agnostic of the (physical) location of that key. For example, usage like `10.0.0.1:2003=4d79d13554fa1301476c1f9fe968b0ac` would allow to change port and/or ip address of the server that receives data for the instance key. Obviously, this way migration of data can be dealt with much more conveniently. The `jump_fnv1a_ch` cluster is also a consistent hash cluster like the previous two, but it does not take the server information into account at all. Whether this is useful to you depends on your scenario. The jump hash has a much better balancing over the servers defined in the cluster, at the expense of not being able to remove any server but the last in order. What this means is that this hash is fine to use with ever growing clusters where older nodes are also replaced at some point. If you have a cluster where removal of old nodes takes place often, the jump hash is not suitable for you. Jump hash works with servers in an ordered list without gaps. To influence the ordering, the instance given to the server will be used as sorting key. Without, the order will be as given in the file. It is a good practice to fix the order of the servers with instances such that it is explicit what the right nodes for the jump hash are. DNS hostnames are resolved to a single address, according to the preference rules in [RFC 3484](https://www.ietf.org/rfc/rfc3484.txt). The `any_of` cluster has an explicit `useall` flag that enables a hostname to resolve to multiple addresses. Each address returned becomes a cluster destination. Match rules are the way to direct incoming metrics to one or more clusters. Match rules are processed top to bottom as they are defined in the file. It is possible to define multiple matches in the same rule. Each match rule can send data to one or more clusters. Since match rules "fall through" unless the `stop` keyword is added, carefully crafted match expression can be used to target multiple clusters or aggregations. This ability allows to replicate metrics, as well as send certain metrics to alternative clusters with careful ordering and usage of the `stop` keyword. The special cluster `blackhole` discards any metrics sent to it. This can be useful for weeding out unwanted metrics in certain cases. Because throwing metrics away is pointless if other matches would accept the same data, a match with as destination the blackhole cluster, has an implicit `stop`. The `validation` clause adds a check to the data (what comes after the metric) in the form of a regular expression. When this expression matches, the match rule will execute as if no validation clause was present. However, if it fails, the match rule is aborted, and no metrics will be sent to destinations, this is the `drop` behaviour. When `log` is used, the metric is logged to stderr. Care should be taken with the latter to avoid log flooding. When a validate clause is present, destinations need not to be present, this allows for applying a global validation rule. Note that the cleansing rules are applied before validation is done, thus the data will not have duplicate spaces. Rewrite rules take a regular input to match incoming metrics, and transform them into the desired new metric name. In the replacement, backreferences are allowed to match capture groups defined in the input regular expression. A match of `server\.(x|y|z)\.` allows to use e.g. `role.\1.` in the substitution. A few caveats apply to the current implementation of rewrite rules. First, their location in the config file determines when the rewrite is performed. The rewrite is done in-place, as such a match rule before the rewrite would match the original name, a match rule after the rewrite no longer matches the original name. Care should be taken with the ordering, as multiple rewrite rules in succession can take place, e.g. `a` gets replaced by `b` and `b` gets replaced by `c` in a succeeding rewrite rule. The second caveat with the current implementation, is that the rewritten metric names are not cleansed, like newly incoming metrics are. Thus, double dots and potential dangerous characters can appear if the replacement string is crafted to produce them. It is the responsibility of the writer to make sure the metrics are clean. If this is an issue for routing, one can consider to have a rewrite-only instance that forwards all metrics to another instance that will do the routing. Obviously the second instance will cleanse the metrics as they come in. The backreference notation allows to lowercase and uppercase the replacement string with the use of the underscore (`_`) and carret (`^`) symbols following directly after the backslash. For example, `role.\_1.` as substitution will lowercase the contents of `\1`. The aggregations defined take one or more input metrics expressed by one or more regular expresions, similar to the match rules. Incoming metrics are aggregated over a period of time defined by the interval in seconds. Since events may arrive a bit later in time, the expiration time in seconds defines when the aggregations should be considered final, as no new entries are allowed to be added any more. On top of an aggregation multiple aggregations can be computed. They can be of the same or different aggregation types, but should write to a unique new metric. The metric names can include back references like in rewrite expressions, allowing for powerful single aggregation rules that yield in many aggregations. When no `send to` clause is given, produced metrics are sent to the relay as if they were submitted from the outside, hence match and aggregation rules apply to those. Care should be taken that loops are avoided this way. For this reason, the use of the `send to` clause is encouraged, to direct the output traffic where possible. Like for match rules, it is possible to define multiple cluster targets. Also, like match rules, the `stop` keyword applies to control the flow of metrics in the matching process. The special `send statistics to` construct is much like a `match` rule which matches the (internal) statistics produced by the relay. It can be used to avoid router loops when sending the statistics to a certain destination. The `send statistics` construct can only be used once, but multiple destinations can be used then required. In case configuration becomes very long, or is managed better in separate files, the `include` directive can be used to read another file. The given file will be read in place and added to the router configuration at the time of inclusion. The end result is one big route configuration. Multiple `include` statements can be used throughout the configuration file. The positioning will influence the order of rules as normal. Beware that recursive inclusion (`include` from an included file) is supported, and currently no safeguards exist for an inclusion loop. For what is worth, this feature likely is best used with simple configuration files (e.g. not having `include` in them). ## EXAMPLES **carbon-c-relay** evolved over time, growing features on demand as the tool proved to be stable and fitting the job well. Below follow some annotated examples of constructs that can be used with the relay. Clusters can be defined as much as necessary. They receive data from match rules, and their type defines which members of the cluster finally get the metric data. The simplest cluster form is a `forward` cluster: ``` cluster send-through forward 10.1.0.1 ; ``` Any metric sent to the `send-through` cluster would simply be forwarded to the server at IPv4 address `10.1.0.1`. If we define multiple servers, all of those servers would get the same metric, thus: ``` cluster send-through forward 10.1.0.1 10.2.0.1 ; ``` The above results in a duplication of metrics send to both machines. This can be useful, but most of the time it is not. The `any_of` cluster type is like `forward`, but it sends each incoming metric to any of the members. The same example with such cluster would be: ``` cluster send-to-any-one any_of 10.1.0.1:2010 10.1.0.1:2011; ``` This would implement a multipath scenario, where two servers are used, the load between them is spread, but should any of them fail, all metrics are sent to the remaining one. This typically works well for upstream relays, or for balancing carbon-cache processes running on the same machine. Should any member become unavailable, for instance due to a rolling restart, the other members receive the traffic. If it is necessary to have true fail-over, where the secondary server is only used if the first is down, the following would implement that: ``` cluster try-first-then-second failover 10.1.0.1:2010 10.1.0.1:2011; ``` These types are different from the two consistent hash cluster types: ``` cluster graphite carbon_ch 127.0.0.1:2006=a 127.0.0.1:2007=b 127.0.0.1:2008=c ; ``` If a member in this example fails, all metrics that would go to that member are kept in the queue, waiting for the member to return. This is useful for clusters of carbon-cache machines where it is desirable that the same metric ends up on the same server always. The `carbon_ch` cluster type is compatible with carbon-relay consistent hash, and can be used for existing clusters populated by carbon-relay. For new clusters, however, it is better to use the `fnv1a_ch` cluster type, for it is faster, and allows to balance over the same address but different ports without an instance number, in constrast to `carbon_ch`. Because we can use multiple clusters, we can also replicate without the use of the `forward` cluster type, in a more intelligent way: ``` cluster dc-old carbon_ch replication 2 10.1.0.1 10.1.0.2 10.1.0.3 ; cluster dc-new1 fnv1a_ch replication 2 10.2.0.1 10.2.0.2 10.2.0.3 ; cluster dc-new2 fnv1a_ch replication 2 10.3.0.1 10.3.0.2 10.3.0.3 ; match * send to dc-old ; match * send to dc-new1 dc-new2 stop ; ``` In this example all incoming metrics are first sent to `dc-old`, then `dc-new1` and finally to `dc-new2`. Note that the cluster type of `dc-old` is different. Each incoming metric will be send to 2 members of all three clusters, thus replicating to in total 6 destinations. For each cluster the destination members are computed independently. Failure of clusters or members does not affect the others, since all have individual queues. The above example could also be written using three match rules for each dc, or one match rule for all three dcs. The difference is mainly in performance, the number of times the incoming metric has to be matched against an expression. The `stop` rule in `dc-new` match rule is not strictly necessary in this example, because there are no more following match rules. However, if the match would target a specific subset, e.g. `^sys\.`, and more clusters would be defined, this could be necessary, as for instance in the following abbreviated example: ``` cluster dc1-sys ... ; cluster dc2-sys ... ; cluster dc1-misc ... ; cluster dc2-misc ... ; match ^sys\. send to dc1-sys; match ^sys\. send to dc2-sys stop; match * send to dc1-misc; match * send to dc2-misc stop; ``` As can be seen, without the `stop` in dc2-sys' match rule, all metrics starting with `sys.` would also be send to dc1-misc and dc2-misc. It can be that this is desired, of course, but in this example there is a dedicated cluster for the `sys` metrics. Suppose there would be some unwanted metric that unfortunately is generated, let's assume some bad/old software. We don't want to store this metric. The `blackhole` cluster is suitable for that, when it is harder to actually whitelist all wanted metrics. Consider the following: ``` match some_legacy1$ some_legacy2$ send to blackhole stop; ``` This would throw away all metrics that end with `some_legacy`, that would otherwise be hard to filter out. Since the order matters, it can be used in a construct like this: ``` cluster old ... ; cluster new ... ; match * send to old; match unwanted send to blackhole stop; match * send to new; ``` In this example the old cluster would receive the metric that's unwanted for the new cluster. So, the order in which the rules occur does matter for the execution. Validation can be used to ensure the data for metrics is as expected. A global validation for just number (no floating point) values could be: ``` match * validate ^[0-9]+\ [0-9]+$ else drop ; ``` (Note the escape with backslash `\` of the space, you might be able to use `\s` or `[:space:]` instead, this depends on your libc implementation.) The validation clause can exist on every match rule, so in principle, the following is valid: ``` match ^foo validate ^[0-9]+\ [0-9]+$ else drop send to integer-cluster ; match ^foo validate ^[0-9.e+-]+\ [0-9.e+-]+$ else drop send to float-cluster stop; ``` Note that the behaviour is different in the previous two examples. When no `send to` clusters are specified, a validation error makes the match behave like the `stop` keyword is present. Likewise, when validation passes, processing continues with the next rule. When destination clusters are present, the `match` respects the `stop` keyword as normal. When specified, processing will always stop when specified so. However, if validation fails, the rule does not send anything to the destination clusters, the metric will be dropped or logged, but never sent. The relay is capable of rewriting incoming metrics on the fly. This process is done based on regular expressions with capture groups that allow to substitute parts in a replacement string. Rewrite rules allow to cleanup metrics from applications, or provide a migration path. In it's simplest form a rewrite rule looks like this: ``` rewrite ^server\.(.+)\.(.+)\.([a-zA-Z]+)([0-9]+) into server.\_1.\2.\3.\3\4 ; ``` In this example a metric like `server.DC.role.name123` would be transformed into `server.dc.role.name.name123`. For rewrite rules hold the same as for matches, that their order matters. Hence to build on top of the old/new cluster example done earlier, the following would store the original metric name in the old cluster, and the new metric name in the new cluster: ``` match * send to old; rewrite ... ; match * send to new; ``` Note that after the rewrite, the original metric name is no longer available, as the rewrite happens in-place. Aggregations are probably the most complex part of carbon-c-relay. Two ways of specifying aggregates are supported by carbon-c-relay. The first, static rules, are handled by an optimiser which tries to fold thousands of rules into groups to make the matching more efficient. The second, dynamic rules, are very powerful compact definitions with possibly thousands of internal instantiations. A typical static aggregation looks like: ``` aggregate ^sys\.dc1\.somehost-[0-9]+\.somecluster\.mysql\.replication_delay ^sys\.dc2\.somehost-[0-9]+\.somecluster\.mysql\.replication_delay every 10 seconds expire after 35 seconds timestamp at end of bucket compute sum write to mysql.somecluster.total_replication_delay compute average write to mysql.somecluster.average_replication_delay compute max write to mysql.somecluster.max_replication_delay compute count write to mysql.somecluster.replication_delay_metric_count ; ``` In this example, four aggregations are produced from the incoming matching metrics. In this example we could have written the two matches as one, but for demonstration purposes we did not. Obviously they can refer to different metrics, if that makes sense. The `every 10 seconds` clause specifies in what interval the aggregator can expect new metrics to arrive. This interval is used to produce the aggregations, thus each 10 seconds 4 new metrics are generated from the data received sofar. Because data may be in transit for some reason, or generation stalled, the `expire after` clause specifies how long the data should be kept before considering a data bucket (which is aggregated) to be complete. In the example, 35 was used, which means after 35 seconds the first aggregates are produced. It also means that metrics can arrive 35 seconds late, and still be taken into account. The exact time at which the aggregate metrics are produced is random between 0 and interval (10 in this case) seconds after the expiry time. This is done to prevent thundering herds of metrics for large aggregation sets. The `timestamp` that is used for the aggregations can be specified to be the `start`, `middle` or `end` of the bucket. Original carbon-aggregator.py uses `start`, while carbon-c-relay's default has always been `end`. The `compute` clauses demonstrate a single aggregation rule can produce multiple aggregates, as often is the case. Internally, this comes for free, since all possible aggregates are always calculated, whether or not they are used. The produced new metrics are resubmitted to the relay, hence matches defined before in the configuration can match output of the aggregator. It is important to avoid loops, that can be generated this way. In general, splitting aggregations to their own carbon-c-relay instance, such that it is easy to forward the produced metrics to another relay instance is a good practice. The previous example could also be written as follows to be dynamic: ``` aggregate ^sys\.dc[0-9].(somehost-[0-9]+)\.([^.]+)\.mysql\.replication_delay every 10 seconds expire after 35 seconds compute sum write to mysql.host.\1.replication_delay compute sum write to mysql.host.all.replication_delay compute sum write to mysql.cluster.\2.replication_delay compute sum write to mysql.cluster.all.replication_delay ; ``` Here a single match, results in four aggregations, each of a different scope. In this example aggregation based on hostname and cluster are being made, as well as the more general `all` targets, which in this example have both identical values. Note that with this single aggregation rule, both per-cluster, per-host and total aggregations are produced. Obviously, the input metrics define which hosts and clusters are produced. With use of the `send to` clause, aggregations can be made more intuitive and less error-prone. Consider the below example: ``` cluster graphite fnv1a_ch ip1 ip2 ip3; aggregate ^sys\.somemetric every 60 seconds expire after 75 seconds compute sum write to sys.somemetric send to graphite stop ; match * send to graphite; ``` It sends all incoming metrics to the graphite cluster, except the sys.somemetric ones, which it replaces with a sum of all the incoming ones. Without a `stop` in the aggregate, this causes a loop, and without the `send to`, the metric name can't be kept its original name, for the output now directly goes to the cluster. ## STATISTICS When **carbon-c-relay** is run without `-d` or `-s` arguments, statistics will be produced and sent to the relay itself in the form of `carbon.relays..*`. The hostname is determined on startup, and can be overriden using the `-H` argument. While many metrics have a similar name to what carbon-cache.py would produce, their values are different. To obtain a more compatible set of values, the `-m` argument can be used to make values non-cumulative, that is, they will report the change compared to the previous value. By default, most values are running counters which only increase over time. The use of the nonNegativeDerivative() function from graphite is useful with these. The default sending interval is 1 minute (60 seconds), but can be overridden using the `-S` argument specified in seconds. The following metrics are produced in the `carbon.relays.` namespace: * metricsReceived The number of metrics that were received by the relay. Received here means that they were seen and processed by any of the dispatchers. * metricsSent The number of metrics that were sent from the relay. This is a total count for all servers combined. When incoming metrics are duplicated by the cluster configuration, this counter will include all those duplications. In other words, the amount of metrics that were successfully sent to other systems. Note that metrics that are processed (received) but still in the sending queue (queued) are not included in this counter. * metricsQueued The total number of metrics that are currently in the queues for all the server targets. This metric is not cumulative, for it is a sample of the queue size, which can (and should) go up and down. Therefore you should not use the derivative function for this metric. * metricsDropped The total number of metric that had to be dropped due to server queues overflowing. A queue typically overflows when the server it tries to send its metrics to is not reachable, or too slow in ingesting the amount of metrics queued. This can be network or resource related, and also greatly depends on the rate of metrics being sent to the particular server. * metricsBlackholed The number of metrics that did not match any rule, or matched a rule with blackhole as target. Depending on your configuration, a high value might be an indication of a misconfiguration somewhere. These metrics were received by the relay, but never sent anywhere, thus they disappeared. * metricStalls The number of times the relay had to stall a client to indicate that the downstream server cannot handle the stream of metrics. A stall is only performed when the queue is full and the server is actually receptive of metrics, but just too slow at the moment. Stalls typically happen during micro-bursts, where the client typically is unaware that it should stop sending more data, while it is able to. * connections The number of connect requests handled. This is an ever increasing number just counting how many connections were accepted. * disconnects The number of disconnected clients. A disconnect either happens because the client goes away, or due to an idle timeout in the relay. The difference between this metric and connections is the amount of connections actively held by the relay. In normal situations this amount remains within reasonable bounds. Many connections, but few disconnections typically indicate a possible connection leak in the client. The idle connections disconnect in the relay here is to guard against resource drain in such scenarios. * dispatch\_wallTime\_us The number of microseconds spent by the dispatchers to do their work. In particular on multi-core systems, this value can be confusing, however, it indicates how long the dispatchers were doing work handling clients. It includes everything they do, from reading data from a socket, cleaning up the input metric, to adding the metric to the appropriate queues. The larger the configuration, and more complex in terms of matches, the more time the dispatchers will spend on the cpu. But also time they do /not/ spend on the cpu is included in this number. It is the pure wallclock time the dispatcher was serving a client. * dispatch\_sleepTime\_us The number of microseconds spent by the dispatchers sleeping waiting for work. When this value gets small (or even zero) the dispatcher has so much work that it doesn't sleep any more, and likely can't process the work in a timely fashion any more. This value plus the wallTime from above sort of sums up to the total uptime taken by this dispatcher. Therefore, expressing the wallTime as percentage of this sum gives the busyness percentage draining all the way up to 100% if sleepTime goes to 0. * server\_wallTime\_us The number of microseconds spent by the servers to send the metrics from their queues. This value includes connection creation, reading from the queue, and sending metrics over the network. * dispatcherX For each indivual dispatcher, the metrics received and blackholed plus the wall clock time. The values are as described above. * destinations.X For all known destinations, the number of dropped, queued and sent metrics plus the wall clock time spent. The values are as described above. * aggregators.metricsReceived The number of metrics that were matched an aggregator rule and were accepted by the aggregator. When a metric matches multiple aggregators, this value will reflect that. A metric is not counted when it is considered syntactically invalid, e.g. no value was found. * aggregators.metricsDropped The number of metrics that were sent to an aggregator, but did not fit timewise. This is either because the metric was too far in the past or future. The expire after clause in aggregate statements controls how long in the past metric values are accepted. * aggregators.metricsSent The number of metrics that were sent from the aggregators. These metrics were produced and are the actual results of aggregations. ## BUGS Please report them at: ## AUTHOR Fabian Groffen <grobian@gentoo.org> ## SEE ALSO All other utilities from the graphite stack. This project aims to be a fast replacement of the original [Carbon relay](http://graphite.readthedocs.org/en/1.0/carbon-daemons.html#carbon-relay-py). **carbon-c-relay** aims to deliver performance and configurability. Carbon is single threaded, and sending metrics to multiple consistent-hash clusters requires chaining of relays. This project provides a multithreaded relay which can address multiple targets and clusters for each and every metric based on pattern matches. There are a couple more replacement projects out there, which are [carbon-relay-ng](https://github.com/graphite-ng/carbon-relay-ng) and [graphite-relay](https://github.com/markchadwick/graphite-relay ). Compared to carbon-relay-ng, this project does provide carbon's consistent-hash routing. graphite-relay, which does this, however doesn't do metric-based matches to direct the traffic, which this project does as well. To date, carbon-c-relay can do aggregations, failover targets and more. ## ACKNOWLEDGEMENTS This program was originally developed for Booking.com. With approval from Booking.com, the code was generalised and published as Open Source on GitHub, for which the author would like to express his gratitude. Author no longer works for Booking.com. carbon-c-relay-2.5/collector.c000066400000000000000000000317561303466152100163250ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include "relay.h" #include "dispatcher.h" #include "server.h" #include "aggregator.h" #include "collector.h" static dispatcher **dispatchers; static char debug = 0; static pthread_t collectorid; static char keep_running = 1; int collector_interval = 60; static char cluster_refresh_pending = 0; static router *pending_router = NULL; /** * Collects metrics from dispatchers and servers and emits them. */ static void * collector_runner(void *s) { int i; size_t totticks; size_t totmetrics; size_t totblackholes; size_t totqueued; size_t totstalls; size_t totdropped; size_t totsleeps; size_t ticks; size_t metrics; size_t blackholes; size_t queued; size_t stalls; size_t dropped; size_t sleeps; time_t now; time_t nextcycle; char destbuf[1024]; /* sort of POSIX_MAX_PATH */ char *p; size_t numaggregators = 0; aggregator *aggrs = NULL; server *submission = (server *)s; server **srvs = NULL; char metric[METRIC_BUFSIZ]; char *m = NULL; size_t sizem = 0; size_t (*s_ticks)(server *); size_t (*s_metrics)(server *); size_t (*s_stalls)(server *); size_t (*s_dropped)(server *); size_t (*d_ticks)(dispatcher *); size_t (*d_metrics)(dispatcher *); size_t (*d_blackholes)(dispatcher *); size_t (*d_sleeps)(dispatcher *); size_t (*a_received)(aggregator *); size_t (*a_sent)(aggregator *); size_t (*a_dropped)(aggregator *); /* setup functions to target what the user wants */ if (debug & 2) { s_ticks = server_get_ticks_sub; s_metrics = server_get_metrics_sub; s_stalls = server_get_stalls_sub; s_dropped = server_get_dropped_sub; d_ticks = dispatch_get_ticks_sub; d_metrics = dispatch_get_metrics_sub; d_blackholes = dispatch_get_blackholes_sub; d_sleeps = dispatch_get_sleeps_sub; a_received = aggregator_get_received_sub; a_sent = aggregator_get_sent_sub; a_dropped = aggregator_get_dropped_sub; } else { s_ticks = server_get_ticks; s_metrics = server_get_metrics; s_stalls = server_get_stalls; s_dropped = server_get_dropped; d_ticks = dispatch_get_ticks; d_metrics = dispatch_get_metrics; d_blackholes = dispatch_get_blackholes; d_sleeps = dispatch_get_sleeps; a_received = aggregator_get_received; a_sent = aggregator_get_sent; a_dropped = aggregator_get_dropped; } #define send(metric) \ if (debug & 1) \ logout("%s", metric); \ else \ server_send(submission, strdup(metric), 1); nextcycle = time(NULL) + collector_interval; while (__sync_bool_compare_and_swap(&keep_running, 1, 1)) { if (cluster_refresh_pending) { char *stub = router_getcollectorstub(pending_router); server **newservers = router_getservers(pending_router); if (srvs != NULL) free(srvs); srvs = newservers; aggrs = router_getaggregators(pending_router); numaggregators = aggregator_numaggregators(aggrs); /* prepare hostname for graphite metrics */ snprintf(metric, sizeof(metric), "%scarbon.relays.%s", stub == NULL ? "" : stub, relay_hostname); for (m = metric + strlen("carbon.relays."); *m != '\0'; m++) if (*m == '.') *m = '_'; *m++ = '.'; *m = '\0'; sizem = sizeof(metric) - (m - metric); cluster_refresh_pending = 0; } assert(srvs != NULL); sleep(1); now = time(NULL); if (nextcycle > now) continue; nextcycle += collector_interval; totticks = 0; totmetrics = 0; totblackholes = 0; totsleeps = 0; for (i = 0; dispatchers[i] != NULL; i++) { totsleeps += sleeps = d_sleeps(dispatchers[i]); totticks += ticks = d_ticks(dispatchers[i]); totmetrics += metrics = d_metrics(dispatchers[i]); totblackholes += blackholes = d_blackholes(dispatchers[i]); snprintf(m, sizem, "dispatcher%d.metricsReceived %zu %zu\n", i + 1, metrics, (size_t)now); send(metric); snprintf(m, sizem, "dispatcher%d.metricsBlackholed %zu %zu\n", i + 1, blackholes, (size_t)now); send(metric); snprintf(m, sizem, "dispatcher%d.wallTime_us %zu %zu\n", i + 1, ticks, (size_t)now); send(metric); snprintf(m, sizem, "dispatcher%d.sleepTime_us %zu %zu\n", i + 1, sleeps, (size_t)now); send(metric); } snprintf(m, sizem, "metricsReceived %zu %zu\n", totmetrics, (size_t)now); send(metric); snprintf(m, sizem, "metricsBlackholed %zu %zu\n", totblackholes, (size_t)now); send(metric); snprintf(m, sizem, "dispatch_wallTime_us %zu %zu\n", totticks, (size_t)now); send(metric); snprintf(m, sizem, "dispatch_sleepTime_us %zu %zu\n", totsleeps, (size_t)now); send(metric); #define send_server_metrics(ipbuf, ticks, metrics, queued, stalls, dropped) \ snprintf(m, sizem, "destinations.%s.sent %zu %zu\n", \ ipbuf, metrics, (size_t)now); \ send(metric); \ snprintf(m, sizem, "destinations.%s.queued %zu %zu\n", \ ipbuf, queued, (size_t)now); \ send(metric); \ snprintf(m, sizem, "destinations.%s.stalls %zu %zu\n", \ ipbuf, stalls, (size_t)now); \ send(metric); \ snprintf(m, sizem, "destinations.%s.dropped %zu %zu\n", \ ipbuf, dropped, (size_t)now); \ send(metric); \ snprintf(m, sizem, "destinations.%s.wallTime_us %zu %zu\n", \ ipbuf, ticks, (size_t)now); \ send(metric); totticks = 0; totmetrics = 0; totqueued = 0; totstalls = 0; totdropped = 0; /* exclude internal_submission metrics from the totals to avoid * artificial doubles due to internal routing details */ ticks = s_ticks(submission); metrics = s_metrics(submission); queued = server_get_queue_len(submission); stalls = s_stalls(submission); dropped = s_dropped(submission); send_server_metrics(server_ip(submission), ticks, metrics, queued, stalls, dropped); for (i = 0; srvs[i] != NULL; i++) { switch (server_ctype(srvs[i])) { case CON_FILE: case CON_PIPE: snprintf(destbuf, sizeof(destbuf), "%s", server_ip(srvs[i])); break; case CON_TCP: snprintf(destbuf, sizeof(destbuf), "%s:%u", server_ip(srvs[i]), server_port(srvs[i])); break; case CON_UDP: snprintf(destbuf, sizeof(destbuf), "%s:%u-udp", server_ip(srvs[i]), server_port(srvs[i])); break; default: assert(0); /* help code analysis tools */ } for (p = destbuf; *p != '\0'; p++) if (*p == '.') *p = '_'; totticks += ticks = s_ticks(srvs[i]); totmetrics += metrics = s_metrics(srvs[i]); totqueued += queued = server_get_queue_len(srvs[i]); totstalls += stalls = s_stalls(srvs[i]); totdropped += dropped = s_dropped(srvs[i]); send_server_metrics(destbuf, ticks, metrics, queued, stalls, dropped); } snprintf(m, sizem, "metricsSent %zu %zu\n", totmetrics, (size_t)now); send(metric); snprintf(m, sizem, "metricsQueued %zu %zu\n", totqueued, (size_t)now); send(metric); snprintf(m, sizem, "metricStalls %zu %zu\n", totstalls, (size_t)now); send(metric); snprintf(m, sizem, "metricsDropped %zu %zu\n", totdropped, (size_t)now); send(metric); snprintf(m, sizem, "server_wallTime_us %zu %zu\n", totticks, (size_t)now); send(metric); snprintf(m, sizem, "connections %zu %zu\n", dispatch_get_accepted_connections(), (size_t)now); send(metric); snprintf(m, sizem, "disconnects %zu %zu\n", dispatch_get_closed_connections(), (size_t)now); send(metric); if (numaggregators > 0) { snprintf(m, sizem, "aggregators.metricsReceived %zu %zu\n", a_received(aggrs), (size_t)now); send(metric); snprintf(m, sizem, "aggregators.metricsSent %zu %zu\n", a_sent(aggrs), (size_t)now); send(metric); snprintf(m, sizem, "aggregators.metricsDropped %zu %zu\n", a_dropped(aggrs), (size_t)now); send(metric); } if (debug & 1) fflush(stdout); } if (srvs != NULL) free(srvs); return NULL; } /** * Writes messages about dropped events or high queue sizes. */ static size_t lastdropped = 0; static size_t lastaggrdropped = 0; static void * collector_writer(void *unused) { int i = 0; size_t queued; size_t queuesize; double queueusage; size_t totdropped; size_t lastconn = 0; size_t lastdisc = 0; size_t numaggregators = 0; server **srvs = NULL; aggregator *aggrs = NULL; while (keep_running) { if (cluster_refresh_pending) { server **newservers = router_getservers(pending_router); if (srvs != NULL) free(srvs); srvs = newservers; aggrs = router_getaggregators(pending_router); numaggregators = aggregator_numaggregators(aggrs); cluster_refresh_pending = 0; } assert(srvs != NULL); sleep(1); if (debug & 1) { size_t mpsout; size_t totout; size_t mpsdrop; size_t totdrop; size_t totqueue; size_t mpsin; size_t totin; size_t totconn; size_t totdisc; size_t dticks; size_t dsleeps; int j; /* Solaris iostat like output: metrics in metrics out metrics drop queue conns disconn workr mps tot mps tot dps tot cur cps tot dps tot act 99999 9999999 99999 9999999 99999 9999999 99999 999 99999 999 99999 99% */ if (i % 24 == 0) printf(" metrics in metrics out metrics drop queue conns disconn workr\n" " mps tot mps tot dps tot cur cps tot dps tot act\n"); mpsout = totout = 0; mpsdrop = totdrop = 0; totqueue = 0; for (j = 0; srvs[j] != NULL; j++) { mpsout += server_get_metrics_sub(srvs[j]); totout += server_get_metrics(srvs[j]); mpsdrop += server_get_dropped_sub(srvs[j]); totdrop += server_get_dropped(srvs[j]); totqueue += server_get_queue_len(srvs[j]); } mpsin = totin = 0; dticks = dsleeps = 0; for (j = 0; dispatchers[j] != NULL; j++) { mpsin += dispatch_get_metrics_sub(dispatchers[j]); totin += dispatch_get_metrics(dispatchers[j]); dticks += dispatch_get_ticks_sub(dispatchers[j]); dsleeps += dispatch_get_sleeps_sub(dispatchers[j]); } totconn = dispatch_get_accepted_connections(); totdisc = dispatch_get_closed_connections(); if (dticks == 0) dticks = 1; /* for Coverity */ printf("%5zu %7zu " /* metrics in */ "%5zu %7zu " /* metrics out */ "%5zu %7zu " /* metrics dropped */ "%5zu " /* queue */ "%3zu %5zu " /* conns */ "%3zu %5zu " /* disconns */ "%2d%%\n", /* workers */ mpsin, totin, mpsout, totout, mpsdrop, totdrop, totqueue, totconn - lastconn, totconn, totdisc - lastdisc, totdisc, (int)(((double)dticks * 100.0) / (double)(dticks + dsleeps)) ); lastconn = totconn; lastdisc = totdisc; } i++; if (i < collector_interval) continue; totdropped = 0; for (i = 0; srvs[i] != NULL; i++) { queued = server_get_queue_len(srvs[i]); queuesize = server_get_queue_size(srvs[i]); totdropped += server_get_dropped(srvs[i]); queueusage = (double)queued / (double)queuesize; if (queueusage >= 0.75) logout("warning: metrics queuing up " "for %s:%u: %zu metrics (%d%% of queue size)\n", server_ip(srvs[i]), server_port(srvs[i]), queued, (int)(queueusage * 100)); } if (totdropped - lastdropped > 0) logout("warning: dropped %zu metrics\n", totdropped - lastdropped); lastdropped = totdropped; if (numaggregators > 0) { totdropped = aggregator_get_dropped(aggrs); if (totdropped - lastaggrdropped > 0) logout("warning: aggregator dropped %zu metrics\n", totdropped - lastaggrdropped); lastaggrdropped = totdropped; } i = 0; } return NULL; } /** * Schedules routes r to be put in place for the current routes. The * replacement is performed at the next cycle of the collector. */ inline void collector_schedulereload(router *rtr) { pending_router = rtr; cluster_refresh_pending = 1; } /** * Returns true if the routes scheduled to be reloaded by a call to * collector_schedulereload() have been activated. */ inline char collector_reloadcomplete(void) { return cluster_refresh_pending == 0; } /** * Initialises and starts the collector. */ void collector_start(dispatcher **d, router *rtr, server *submission, char cum) { dispatchers = d; collector_schedulereload(rtr); if (mode & MODE_DEBUG) debug = 1; debug |= (cum ? 0 : 2); if (mode & MODE_SUBMISSION) { if (pthread_create(&collectorid, NULL, collector_writer, NULL) != 0) logerr("failed to start collector!\n"); } else { if (pthread_create(&collectorid, NULL, collector_runner, submission) != 0) logerr("failed to start collector!\n"); } } /** * Shuts down the collector. */ void collector_stop(void) { __sync_bool_compare_and_swap(&keep_running, 1, 0); pthread_join(collectorid, NULL); } carbon-c-relay-2.5/collector.h000066400000000000000000000021441303466152100163170ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef COLLECTOR_H #define COLLECTOR_H 1 #include "dispatcher.h" #include "router.h" #include "aggregator.h" #include "server.h" #include "relay.h" extern int collector_interval; #define timediff(X, Y) \ (Y.tv_sec > X.tv_sec ? (Y.tv_sec - X.tv_sec) * 1000 * 1000 + ((Y.tv_usec - X.tv_usec)) : Y.tv_usec - X.tv_usec) void collector_start(dispatcher **d, router *rtr, server *submission, char cum); void collector_stop(void); void collector_schedulereload(router *rtr); char collector_reloadcomplete(void); #endif carbon-c-relay-2.5/consistent-hash.c000066400000000000000000000301631303466152100174400ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include "fnv1a.h" #include "md5.h" #include "server.h" #define CH_RING struct _ch_ring #include "consistent-hash.h" /* This value is hardwired in the carbon sources, and necessary to get * fair (re)balancing of metrics in the hash ring. Because the value * seems reasonable, we use the same value for carbon and fnv1a hash * implementations. */ #define HASH_REPLICAS 100 typedef struct _ring_entry { unsigned short pos; unsigned char malloced:1; server *server; struct _ring_entry *next; } ch_ring_entry; struct _ch_ring { ch_type type; unsigned char hash_replicas; ch_ring_entry *entries; ch_ring_entry **entrylist; /* only used with jump hash */ int entrycnt; }; /** * Computes the hash position for key in a 16-bit unsigned integer * space. Returns a number between 0 and 65535 based on the highest 2 * bytes of the MD5 sum of key. */ static unsigned short carbon_hashpos(const char *key, const char *end) { unsigned char md5[MD5_DIGEST_LENGTH]; MD5((unsigned char *)key, end - key, md5); return ((md5[0] << 8) + md5[1]); } /** * Computes the hash position for key in a 16-bit unsigned integer * space. Returns a number between 0 and 65535 based on the FNV1a hash * algorithm. */ static unsigned short fnv1a_hashpos(const char *key, const char *end) { unsigned int hash; fnv1a_32(hash, key, key, end); return (unsigned short)((hash >> 16) ^ (hash & (unsigned int)0xFFFF)); } /** * Computes the bucket number for key in the range [0, bckcnt). The * algorithm used is the jump consistent hash by Lamping and Veach. */ static unsigned int jump_bucketpos(unsigned long long int key, int bckcnt) { long long int b = -1, j = 0; while (j < bckcnt) { b = j; key = key * 2862933555777941757ULL + 1; j = (long long int)((double)(b + 1) * ((double)(1LL << 31) / (double)((key >> 33) + 1)) ); } /* b cannot exceed the range of bckcnt, see while condition */ return (unsigned int)b; } /** * Sort comparator for ch_ring_entry structs on pos, ip and instance. */ static int entrycmp_carbon(const void *l, const void *r) { ch_ring_entry *ch_l = (ch_ring_entry *)l; ch_ring_entry *ch_r = (ch_ring_entry *)r; if (ch_l->pos != ch_r->pos) return ch_l->pos - ch_r->pos; #ifndef CH_CMP_V40_BEHAVIOUR { int d = strcmp(server_ip(ch_l->server), server_ip(ch_r->server)); char *i_l, *i_r; if (d != 0) return d; i_l = server_instance(ch_l->server); i_r = server_instance(ch_r->server); if (i_l == NULL && i_r == NULL) return 0; if (i_l == NULL) return 1; if (i_r == NULL) return -1; return strcmp(i_l, i_r); } #endif return 0; } /** * Sort comparator for ch_ring_entry structs on pos, ip and port. */ static int entrycmp_fnv1a(const void *l, const void *r) { ch_ring_entry *ch_l = (ch_ring_entry *)l; ch_ring_entry *ch_r = (ch_ring_entry *)r; if (ch_l->pos != ch_r->pos) return ch_l->pos - ch_r->pos; #ifndef CH_CMP_V40_BEHAVIOUR { int d = strcmp(server_ip(ch_l->server), server_ip(ch_r->server)); if (d != 0) return d; return server_port(ch_l->server) - server_port(ch_r->server); } #endif return 0; } /** * Sort comparator for ch_ring_entry structs on instance only. */ static int entrycmp_jump_fnv1a(const void *l, const void *r) { char *si_l = server_instance(((ch_ring_entry *)l)->server); char *si_r = server_instance(((ch_ring_entry *)r)->server); return strcmp(si_l ? si_l : "", si_r ? si_r : ""); } ch_ring * ch_new(ch_type type) { ch_ring *ret = malloc(sizeof(ch_ring)); if (ret == NULL) return NULL; ret->type = type; switch (ret->type) { case CARBON: case FNV1a: ret->hash_replicas = HASH_REPLICAS; break; default: ret->hash_replicas = 1; break; } ret->entries = NULL; ret->entrylist = NULL; ret->entrycnt = 0; return ret; } /** * Computes the hash positions for the server name given. This is based * on the hashpos function. The server name usually is the IPv4 * address. The port component is just stored and not used in the * carbon hash calculation in case of carbon_ch. The instance component * is used in the hash calculation of carbon_ch, it is ignored for * fnv1a_ch. Returns an updated ring. */ ch_ring * ch_addnode(ch_ring *ring, server *s) { int i; char buf[256]; ch_ring_entry *entries; char *instance = server_instance(s); int (*cmp)(const void *, const void *) = NULL; if (ring == NULL) return NULL; entries = (ch_ring_entry *)malloc(sizeof(ch_ring_entry) * ring->hash_replicas); if (entries == NULL) return NULL; switch (ring->type) { case CARBON: for (i = 0; i < ring->hash_replicas; i++) { /* this format is actually Python's tuple format that is * used in serialised form as input for the hash */ snprintf(buf, sizeof(buf), "('%s', %s%s%s):%d", server_ip(s), instance == NULL ? "" : "'", instance == NULL ? "None" : instance, instance == NULL ? "" : "'", i); /* carbon upstream committed: * https://github.com/graphite-project/carbon/commit/024f9e67ca47619438951c59154c0dec0b0518c7 * this makes sure no collissions exist on pos, however, * at the expense of being agnostic to the input order, * therefore that change isn't implemented here, see * https://github.com/grobian/carbon-c-relay/issues/84 */ entries[i].pos = carbon_hashpos(buf, buf + strlen(buf)); entries[i].server = s; entries[i].next = NULL; entries[i].malloced = 0; } cmp = *entrycmp_carbon; break; case FNV1a: for (i = 0; i < ring->hash_replicas; i++) { /* take all server info into account, such that * different port numbers for the same hosts will work * (unlike CARBON), unless we got a full overrride */ if (instance == NULL) { snprintf(buf, sizeof(buf), "%d-%s:%u", i, server_ip(s), server_port(s)); } else { snprintf(buf, sizeof(buf), "%d-%s", i, instance); } entries[i].pos = fnv1a_hashpos(buf, buf + strlen(buf)); entries[i].server = s; entries[i].next = NULL; entries[i].malloced = 0; } cmp = *entrycmp_fnv1a; break; case JUMP_FNV1a: entries[0].pos = 0; entries[0].server = s; entries[0].next = NULL; entries[0].malloced = 0; cmp = *entrycmp_jump_fnv1a; break; } /* sort to allow merge joins later down the road */ qsort(entries, ring->hash_replicas, sizeof(ch_ring_entry), cmp); entries[0].malloced = 1; if (ring->entries == NULL) { for (i = 1; i < ring->hash_replicas; i++) entries[i - 1].next = &entries[i]; ring->entries = entries; } else { /* merge-join the two rings */ ch_ring_entry *w, *last; i = 0; last = NULL; assert(ring->hash_replicas > 0); for (w = ring->entries; w != NULL && i < ring->hash_replicas; ) { if (cmp(w, &entries[i]) <= 0) { last = w; w = w->next; } else { entries[i].next = w; if (last == NULL) { ring->entries = &entries[i]; } else { last->next = &entries[i]; } last = &entries[i]; i++; } } if (w != NULL) { last->next = w; } else { last->next = &entries[i]; for (i = i + 1; i < ring->hash_replicas; i++) entries[i - 1].next = &entries[i]; } } if (ring->type == JUMP_FNV1a) { ch_ring_entry *w; /* count the ring, pos is purely cosmetic, it isn't used */ for (w = ring->entries, i = 0; w != NULL; w = w->next, i++) w->pos = i; ring->entrycnt = i; /* this is really wasteful, but optimising this isn't worth it * since it's called only a few times during config parsing */ if (ring->entrylist != NULL) free(ring->entrylist); ring->entrylist = malloc(sizeof(ch_ring_entry *) * ring->entrycnt); for (w = ring->entries, i = 0; w != NULL; w = w->next, i++) ring->entrylist[i] = w; if (i == CONN_DESTS_SIZE) { logerr("ch_addnode: nodes in use exceeds CONN_DESTS_SIZE, " "increase CONN_DESTS_SIZE in router.h\n"); return NULL; } } return ring; } /** * Retrieve the nodes responsible for storing the given metric. The * replcnt argument specifies how many hosts should be retrieved. * Results are stored in ret, an array of ch_ring pointers. The * caller is responsible for ensuring that ret is large enough to store * replcnt pointers. */ void ch_get_nodes( destination ret[], ch_ring *ring, const char replcnt, const char *metric, const char *firstspace) { ch_ring_entry *w; unsigned short pos = 0; int i, j; switch (ring->type) { case CARBON: pos = carbon_hashpos(metric, firstspace); break; case FNV1a: pos = fnv1a_hashpos(metric, firstspace); break; case JUMP_FNV1a: { /* this is really a short route, since the jump hash gives * us a bucket immediately */ unsigned long long int hash; ch_ring_entry *bcklst[CONN_DESTS_SIZE]; const char *p; i = ring->entrycnt; pos = replcnt; memcpy(bcklst, ring->entrylist, sizeof(bcklst[0]) * i); fnv1a_64(hash, p, metric, firstspace); while (i > 0) { j = jump_bucketpos(hash, i); (*ret).dest = bcklst[j]->server; (*ret).metric = strdup(metric); ret++; if (--pos == 0) break; /* use xorshift to generate a different hash for input * in the hump hash again */ hash ^= hash >> 12; hash ^= hash << 25; hash ^= hash >> 27; hash *= 2685821657736338717ULL; /* remove the server we just selected, such that we can * be sure the next iteration will fetch another server */ bcklst[j] = bcklst[--i]; } } return; } assert(ring->entries); /* implement behaviour of Python's bisect_left on the ring (used in * carbon hash source), one day we might want to implement it as * real binary search iso forward pointer chasing */ for (w = ring->entries, i = 0; w != NULL; i++, w = w->next) if (w->pos >= pos) break; /* now fetch enough unique servers to match the requested count */ for (i = 0; i < replcnt; i++, w = w->next) { if (w == NULL) w = ring->entries; for (j = i - 1; j >= 0; j--) { if (ret[j].dest == w->server) { j = i; break; } } if (j == i) { i--; continue; } ret[i].dest = w->server; ret[i].metric = strdup(metric); } } void ch_printhashring(ch_ring *ring, FILE *f) { ch_ring_entry *w; char column = 0; char srvbuf[21]; for (w = ring->entries; w != NULL; w = w->next) { snprintf(srvbuf, sizeof(srvbuf), "%s:%d%s%s", server_ip(w->server), server_port(w->server), server_instance(w->server) ? "=" : "", server_instance(w->server) ? server_instance(w->server) : ""); fprintf(f, "%5d@%-20s", w->pos, srvbuf); if (column < 2) { fprintf(f, " "); column++; } else { fprintf(f, "\n"); column = 0; } } if (column != 0) fprintf(f, "\n"); } unsigned short ch_gethashpos(ch_ring *ring, const char *key, const char *end) { switch (ring->type) { case CARBON: return carbon_hashpos(key, end); case FNV1a: return fnv1a_hashpos(key, end); case JUMP_FNV1a: { unsigned long long int hash; fnv1a_64(hash, key, key, end); return jump_bucketpos(hash, ring->entrycnt); } default: assert(0); /* this shouldn't happen */ } return 0; /* pacify compiler */ } /** * Frees the ring structure and its added nodes, leaves the referenced * servers untouched. */ void ch_free(ch_ring *ring) { ch_ring_entry *deletes = NULL; ch_ring_entry *w = NULL; for (; ring->entries != NULL; ring->entries = ring->entries->next) { if (ring->entries->malloced) { if (deletes == NULL) { w = deletes = ring->entries; } else { w = w->next = ring->entries; } } } if (w != NULL) { w->next = NULL; while (deletes != NULL) { w = deletes->next; free(deletes); deletes = w; } } if (ring->entrylist != NULL) free(ring->entrylist); free(ring); } carbon-c-relay-2.5/consistent-hash.h000066400000000000000000000022761303466152100174510ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef CONSISTENT_HASH_H #define CONSISTENT_HASH_H 1 #include #include "server.h" #include "router.h" #ifndef CH_RING #define CH_RING void #endif typedef CH_RING ch_ring; typedef enum { CARBON, FNV1a, JUMP_FNV1a } ch_type; ch_ring *ch_new(ch_type type); ch_ring *ch_addnode(ch_ring *ring, server *s); void ch_get_nodes( destination ret[], ch_ring *ring, const char replcnt, const char *metric, const char *firstspace); void ch_printhashring(ch_ring *ring, FILE *out); unsigned short ch_gethashpos(ch_ring *ring, const char *key, const char *end); void ch_free(ch_ring *ring); #endif carbon-c-relay-2.5/contrib/000077500000000000000000000000001303466152100156175ustar00rootroot00000000000000carbon-c-relay-2.5/contrib/relay.logrotate000066400000000000000000000004341303466152100206560ustar00rootroot00000000000000# not installed by default as logrotate is used to manage all carbon log files. /var/log/%%NAME%%/%%NAME%%.log { sharedscripts missingok notifempty rotate 30 compress postrotate [ ! -f /var/run/%%NAME%%/%%NAME%%.pid ] || /etc/init.d/%%NAME%% restart endscript } carbon-c-relay-2.5/contrib/relay.monit000066400000000000000000000004621303466152100200050ustar00rootroot00000000000000# Monit script to ensure carbon c relay is always running check process %%NAME%% with pidfile /var/run/%%NAME%%/%%NAME%%.pid start program = "/etc/init.d/%%NAME%% start" stop program = "/etc/init.d/%%NAME%% stop" if failed port 2003 type tcp then restart if 5 restarts within 5 cycles then timeout carbon-c-relay-2.5/dispatcher.c000066400000000000000000000543151303466152100164610ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "relay.h" #include "router.h" #include "server.h" #include "collector.h" #include "dispatcher.h" enum conntype { LISTENER, CONNECTION }; typedef struct _connection { int sock; char takenby; /* -2: being setup, -1: free, 0: not taken, >0: tid */ char srcaddr[24]; /* string representation of source address */ char buf[METRIC_BUFSIZ]; int buflen; char needmore:1; char noexpire:1; char metric[METRIC_BUFSIZ]; destination dests[CONN_DESTS_SIZE]; size_t destlen; struct timeval lastwork; unsigned int maxsenddelay; char hadwork:1; char isaggr:1; char isudp:1; } connection; struct _dispatcher { pthread_t tid; enum conntype type; char id; size_t metrics; size_t blackholes; size_t ticks; size_t sleeps; size_t prevmetrics; size_t prevblackholes; size_t prevticks; size_t prevsleeps; char keep_running; /* full byte for atomic access */ router *rtr; router *pending_rtr; char route_refresh_pending; /* full byte for atomic access */ char hold:1; char *allowed_chars; }; static connection **listeners = NULL; static connection *connections = NULL; static size_t connectionslen = 0; pthread_rwlock_t listenerslock = PTHREAD_RWLOCK_INITIALIZER; pthread_rwlock_t connectionslock = PTHREAD_RWLOCK_INITIALIZER; static size_t acceptedconnections = 0; static size_t closedconnections = 0; static unsigned int sockbufsize = 0; /** * Helper function to try and be helpful to the user. If errno * indicates no new fds could be made, checks what the current max open * files limit is, and if it's close to what we have in use now, write * an informative message to stderr. */ void dispatch_check_rlimit_and_warn(void) { if (errno == EISCONN || errno == EMFILE) { struct rlimit ofiles; /* rlimit can be changed for the running process (at least on * Linux 2.6+) so refetch this value every time, should only * occur on errors anyway */ if (getrlimit(RLIMIT_NOFILE, &ofiles) < 0) ofiles.rlim_max = 0; if (ofiles.rlim_max != RLIM_INFINITY && ofiles.rlim_max > 0) logerr("process configured maximum connections = %d, " "consider raising max open files/max descriptor limit\n", (int)ofiles.rlim_max); } } #define MAX_LISTENERS 32 /* hopefully enough */ /** * Adds an (initial) listener socket to the chain of connections. * Listener sockets are those which need to be accept()-ed on. */ int dispatch_addlistener(int sock) { connection *newconn; int c; pthread_rwlock_wrlock(&listenerslock); if (listeners == NULL) { if ((listeners = malloc(sizeof(connection *) * MAX_LISTENERS)) == NULL) { pthread_rwlock_unlock(&listenerslock); return 1; } for (c = 0; c < MAX_LISTENERS; c++) listeners[c] = NULL; } for (c = 0; c < MAX_LISTENERS; c++) { if (listeners[c] == NULL) { newconn = malloc(sizeof(connection)); if (newconn == NULL) { pthread_rwlock_unlock(&listenerslock); return 1; } (void) fcntl(sock, F_SETFL, O_NONBLOCK); newconn->sock = sock; newconn->takenby = 0; newconn->buflen = 0; listeners[c] = newconn; break; } } if (c == MAX_LISTENERS) { logerr("cannot add new listener: " "no more free listener slots (max = %zu)\n", MAX_LISTENERS); pthread_rwlock_unlock(&listenerslock); return 1; } pthread_rwlock_unlock(&listenerslock); return 0; } /** * Remove listener from the listeners list. Each removal will incur a * global lock. Frequent usage of this function is not anticipated. */ void dispatch_removelistener(int sock) { int c; connection *conn; pthread_rwlock_wrlock(&listenerslock); /* find connection */ for (c = 0; c < MAX_LISTENERS; c++) if (listeners[c] != NULL && listeners[c]->sock == sock) break; if (c == MAX_LISTENERS) { /* not found?!? */ logerr("dispatch: cannot find listener!\n"); pthread_rwlock_unlock(&listenerslock); return; } /* cleanup */ conn = listeners[c]; close(conn->sock); free(conn); listeners[c] = NULL; pthread_rwlock_unlock(&listenerslock); } #define CONNGROWSZ 1024 /** * Adds a connection socket to the chain of connections. * Connection sockets are those which need to be read from. * Returns the connection id, or -1 if a failure occurred. */ int dispatch_addconnection(int sock) { size_t c; struct sockaddr_in6 saddr; socklen_t saddr_len = sizeof(saddr); pthread_rwlock_rdlock(&connectionslock); for (c = 0; c < connectionslen; c++) if (__sync_bool_compare_and_swap(&(connections[c].takenby), -1, -2)) break; pthread_rwlock_unlock(&connectionslock); if (c == connectionslen) { connection *newlst; pthread_rwlock_wrlock(&connectionslock); if (connectionslen > c) { /* another dispatcher just extended the list */ pthread_rwlock_unlock(&connectionslock); return dispatch_addconnection(sock); } newlst = realloc(connections, sizeof(connection) * (connectionslen + CONNGROWSZ)); if (newlst == NULL) { logerr("cannot add new connection: " "out of memory allocating more slots (max = %zu)\n", connectionslen); pthread_rwlock_unlock(&connectionslock); return -1; } memset(&newlst[connectionslen], '\0', sizeof(connection) * CONNGROWSZ); for (c = connectionslen; c < connectionslen + CONNGROWSZ; c++) newlst[c].takenby = -1; /* free */ connections = newlst; c = connectionslen; /* for the setup code below */ newlst[c].takenby = -2; connectionslen += CONNGROWSZ; pthread_rwlock_unlock(&connectionslock); } /* figure out who's calling */ if (getpeername(sock, (struct sockaddr *)&saddr, &saddr_len) == 0) { snprintf(connections[c].srcaddr, sizeof(connections[c].srcaddr), "(unknown)"); switch (saddr.sin6_family) { case PF_INET: inet_ntop(saddr.sin6_family, &((struct sockaddr_in *)&saddr)->sin_addr, connections[c].srcaddr, sizeof(connections[c].srcaddr)); break; case PF_INET6: inet_ntop(saddr.sin6_family, &saddr.sin6_addr, connections[c].srcaddr, sizeof(connections[c].srcaddr)); break; } } (void) fcntl(sock, F_SETFL, O_NONBLOCK); if (sockbufsize > 0) { if (setsockopt(sock, SOL_SOCKET, SO_RCVBUF, &sockbufsize, sizeof(sockbufsize)) != 0) ; } connections[c].sock = sock; connections[c].buflen = 0; connections[c].needmore = 0; connections[c].noexpire = 0; connections[c].isaggr = 0; connections[c].isudp = 0; connections[c].destlen = 0; gettimeofday(&connections[c].lastwork, NULL); connections[c].hadwork = 1; /* force first iteration before stalling */ /* after this dispatchers will pick this connection up */ __sync_bool_compare_and_swap(&(connections[c].takenby), -2, 0); __sync_add_and_fetch(&acceptedconnections, 1); return c; } /** * Adds a connection which we know is from an aggregator, so direct * pipe. This is different from normal connections that we don't want * to count them, never expire them, and want to recognise them when * we're doing reloads. */ int dispatch_addconnection_aggr(int sock) { int conn = dispatch_addconnection(sock); if (conn == -1) return 1; connections[conn].noexpire = 1; connections[conn].isaggr = 1; acceptedconnections--; return 0; } /** * Adds a pseudo-listener for datagram (UDP) sockets, which is pseudo, * for in fact it adds a new connection, but makes sure that connection * won't be closed after being idle, and won't count that connection as * an incoming connection either. */ int dispatch_addlistener_udp(int sock) { int conn = dispatch_addconnection(sock); if (conn == -1) return 1; connections[conn].noexpire = 1; connections[conn].isudp = 1; acceptedconnections--; return 0; } inline static char dispatch_process_dests(connection *conn, dispatcher *self, struct timeval now) { int i; char force; if (conn->destlen > 0) { if (conn->maxsenddelay == 0) conn->maxsenddelay = ((rand() % 750) + 250) * 1000; /* force after timeout */ force = timediff(conn->lastwork, now) > conn->maxsenddelay; for (i = 0; i < conn->destlen; i++) { tracef("dispatcher %d, connfd %d, metric %s, queueing to %s:%d\n", self->id, conn->sock, conn->dests[i].metric, server_ip(conn->dests[i].dest), server_port(conn->dests[i].dest)); if (server_send(conn->dests[i].dest, conn->dests[i].metric, force) == 0) break; } if (i != conn->destlen) { conn->destlen -= i; memmove(&conn->dests[0], &conn->dests[i], (sizeof(destination) * conn->destlen)); return 0; } else { /* finally "complete" this metric */ conn->destlen = 0; conn->lastwork = now; conn->hadwork = 1; } } return 1; } #define IDLE_DISCONNECT_TIME (10 * 60 * 1000 * 1000) /* 10 minutes */ /** * Look at conn and see if works needs to be done. If so, do it. This * function operates on an (exclusive) lock on the connection it serves. * Schematically, what this function does is like this: * * read (partial) data <---- * | | * v | * split and clean metrics | * | | * v | * route metrics | feedback loop * | | (stall client) * v | * send 1st attempt | * \ | * v* | * this is optional, but if a server's * retry send (<1s) -------- queue is full, the client is stalled * block reads */ static int dispatch_connection(connection *conn, dispatcher *self, struct timeval start) { char *p, *q, *firstspace, *lastnl; int len; /* first try to resume any work being blocked */ if (dispatch_process_dests(conn, self, start) == 0) { __sync_bool_compare_and_swap(&(conn->takenby), self->id, 0); return 0; } /* don't poll (read) when the last time we ran nothing happened, * this is to avoid excessive CPU usage, issue #126 */ if (!conn->hadwork && timediff(conn->lastwork, start) < 100 * 1000) { __sync_bool_compare_and_swap(&(conn->takenby), self->id, 0); return 0; } conn->hadwork = 0; len = -2; /* try to read more data, if that succeeds, or we still have data * left in the buffer, try to process the buffer */ if ( (!conn->needmore && conn->buflen > 0) || (len = read(conn->sock, conn->buf + conn->buflen, (sizeof(conn->buf) - 1) - conn->buflen)) > 0 ) { if (len > 0) { conn->buflen += len; tracef("dispatcher %d, connfd %d, read %d bytes from socket\n", self->id, conn->sock, len); } /* metrics look like this: metric_path value timestamp\n * due to various messups we need to sanitise the * metrics_path here, to ensure we can calculate the metric * name off the filesystem path (and actually retrieve it in * the web interface). */ q = conn->metric; firstspace = NULL; lastnl = NULL; for (p = conn->buf; p - conn->buf < conn->buflen; p++) { if (*p == '\n' || *p == '\r') { /* end of metric */ lastnl = p; /* just a newline on it's own? some random garbage? skip */ if (q == conn->metric || firstspace == NULL) { q = conn->metric; firstspace = NULL; continue; } __sync_add_and_fetch(&(self->metrics), 1); *q++ = '\n'; *q = '\0'; /* can do this because we substract one from buf */ /* perform routing of this metric */ tracef("dispatcher %d, connfd %d, metric %s", self->id, conn->sock, conn->metric); __sync_add_and_fetch(&(self->blackholes), router_route(self->rtr, conn->dests, &conn->destlen, CONN_DESTS_SIZE, conn->srcaddr, conn->metric, firstspace)); tracef("dispatcher %d, connfd %d, destinations %zd\n", self->id, conn->sock, conn->destlen); /* restart building new one from the start */ q = conn->metric; firstspace = NULL; conn->hadwork = 1; gettimeofday(&conn->lastwork, NULL); conn->maxsenddelay = 0; /* send the metric to where it is supposed to go */ if (dispatch_process_dests(conn, self, conn->lastwork) == 0) break; } else if (*p == ' ' || *p == '\t' || *p == '.') { /* separator */ if (q == conn->metric) { /* make sure we skip this on next iteration to * avoid an infinite loop, issues #8 and #51 */ lastnl = p; continue; } if (*p == '\t') *p = ' '; if (*p == ' ' && firstspace == NULL) { if (*(q - 1) == '.') q--; /* strip trailing separator */ firstspace = q; *q++ = ' '; } else { /* metric_path separator or space, * - duplicate elimination * - don't start with separator/space */ if (*(q - 1) != *p && (q - 1) != firstspace) *q++ = *p; } } else if (firstspace != NULL || (*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || (*p >= '0' && *p <= '9') || strchr(self->allowed_chars, *p)) { /* copy char */ *q++ = *p; } else { /* something barf, replace by underscore */ *q++ = '_'; } } conn->needmore = q != conn->metric; if (lastnl != NULL) { /* move remaining stuff to the front */ conn->buflen -= lastnl + 1 - conn->buf; memmove(conn->buf, lastnl + 1, conn->buflen); } } if (len == -1 && (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)) { /* nothing available/no work done */ struct timeval stop; gettimeofday(&stop, NULL); if (!conn->noexpire && timediff(conn->lastwork, stop) > IDLE_DISCONNECT_TIME) { /* force close connection below */ len = 0; } else { __sync_bool_compare_and_swap(&(conn->takenby), self->id, 0); return 0; } } if (len == -1 || len == 0 || conn->isudp) { /* error + EOF */ /* we also disconnect the client in this case if our reading * buffer is full, but we still need more (read returns 0 if the * size argument is 0) -> this is good, because we can't do much * with such client */ if (conn->noexpire) { /* reset buffer only (UDP/aggregations) and move on */ conn->needmore = 1; conn->buflen = 0; __sync_bool_compare_and_swap(&(conn->takenby), self->id, 0); return len > 0; } else { __sync_add_and_fetch(&closedconnections, 1); close(conn->sock); /* flag this connection as no longer in use */ __sync_bool_compare_and_swap(&(conn->takenby), self->id, -1); return 0; } } /* "release" this connection again */ __sync_bool_compare_and_swap(&(conn->takenby), self->id, 0); return 1; } /** * pthread compatible routine that handles connections and processes * whatever comes in on those. */ static void * dispatch_runner(void *arg) { dispatcher *self = (dispatcher *)arg; connection *conn; int c; if (self->type == LISTENER) { struct pollfd ufds[MAX_LISTENERS]; int fds; while (__sync_bool_compare_and_swap(&(self->keep_running), 1, 1)) { pthread_rwlock_rdlock(&listenerslock); fds = 0; for (c = 0; c < MAX_LISTENERS; c++) { if (listeners[c] == NULL) continue; ufds[fds].fd = listeners[c]->sock; ufds[fds].events = POLLIN; fds++; } pthread_rwlock_unlock(&listenerslock); if (poll(ufds, fds, 1000) > 0) { for (c = fds - 1; c >= 0; c--) { if (ufds[c].revents & POLLIN) { int client; struct sockaddr addr; socklen_t addrlen = sizeof(addr); if ((client = accept(ufds[c].fd, &addr, &addrlen)) < 0) { logerr("dispatch: failed to " "accept() new connection: %s\n", strerror(errno)); dispatch_check_rlimit_and_warn(); continue; } if (dispatch_addconnection(client) == -1) { close(client); continue; } } } } } } else if (self->type == CONNECTION) { int work; struct timeval start, stop; while (__sync_bool_compare_and_swap(&(self->keep_running), 1, 1)) { work = 0; if (__sync_bool_compare_and_swap(&(self->route_refresh_pending), 1, 1)) { self->rtr = self->pending_rtr; self->pending_rtr = NULL; __sync_bool_compare_and_swap(&(self->route_refresh_pending), 1, 0); self->hold = 0; } gettimeofday(&start, NULL); pthread_rwlock_rdlock(&connectionslock); for (c = 0; c < connectionslen; c++) { conn = &(connections[c]); /* atomically try to "claim" this connection */ if (!__sync_bool_compare_and_swap( &(conn->takenby), 0, self->id)) continue; if (self->hold && !conn->isaggr) { __sync_bool_compare_and_swap( &(conn->takenby), self->id, 0); continue; } work += dispatch_connection(conn, self, start); } pthread_rwlock_unlock(&connectionslock); gettimeofday(&stop, NULL); __sync_add_and_fetch(&(self->ticks), timediff(start, stop)); /* nothing done, avoid spinlocking */ if (__sync_bool_compare_and_swap(&(self->keep_running), 1, 1) && work == 0) { gettimeofday(&start, NULL); usleep((100 + (rand() % 200)) * 1000); /* 100ms - 300ms */ gettimeofday(&stop, NULL); __sync_add_and_fetch(&(self->sleeps), timediff(start, stop)); } } } else { logerr("huh? unknown self type!\n"); } return NULL; } /** * Starts a new dispatcher for the given type and with the given id. * Returns its handle. */ static dispatcher * dispatch_new(char id, enum conntype type, router *r, char *allowed_chars) { dispatcher *ret = malloc(sizeof(dispatcher)); if (ret == NULL) return NULL; ret->id = id; ret->type = type; ret->keep_running = 1; ret->rtr = r; ret->route_refresh_pending = 0; ret->hold = 0; ret->allowed_chars = allowed_chars; ret->metrics = 0; ret->blackholes = 0; ret->ticks = 0; ret->sleeps = 0; ret->prevmetrics = 0; ret->prevblackholes = 0; ret->prevticks = 0; ret->prevsleeps = 0; if (pthread_create(&ret->tid, NULL, dispatch_runner, ret) != 0) { free(ret); return NULL; } return ret; } void dispatch_set_bufsize(unsigned int nsockbufsize) { sockbufsize = nsockbufsize; } static char globalid = 0; /** * Starts a new dispatcher specialised in handling incoming connections * (and putting them on the queue for handling the connections). */ dispatcher * dispatch_new_listener(void) { char id = globalid++; return dispatch_new(id, LISTENER, NULL, NULL); } /** * Starts a new dispatcher specialised in handling incoming data on * existing connections. */ dispatcher * dispatch_new_connection(router *r, char *allowed_chars) { char id = globalid++; return dispatch_new(id, CONNECTION, r, allowed_chars); } /** * Signals this dispatcher to stop whatever it's doing. */ void dispatch_stop(dispatcher *d) { __sync_bool_compare_and_swap(&(d->keep_running), 1, 0); } /** * Shuts down dispatcher d. Returns when the dispatcher has terminated. */ void dispatch_shutdown(dispatcher *d) { dispatch_stop(d); pthread_join(d->tid, NULL); } /** * Free up resources taken by dispatcher d. The caller should make sure * the dispatcher has been shut down at this point. */ void dispatch_free(dispatcher *d) { free(d); } /** * Requests this dispatcher to stop processing connections. As soon as * schedulereload finishes reloading the routes, this dispatcher will * un-hold and continue processing connections. * Returns when the dispatcher is no longer doing work. */ inline void dispatch_hold(dispatcher *d) { d->hold = 1; } /** * Schedules routes r to be put in place for the current routes. The * replacement is performed at the next cycle of the dispatcher. */ inline void dispatch_schedulereload(dispatcher *d, router *r) { d->pending_rtr = r; __sync_bool_compare_and_swap(&(d->route_refresh_pending), 0, 1); } /** * Returns true if the routes scheduled to be reloaded by a call to * dispatch_schedulereload() have been activated. */ inline char dispatch_reloadcomplete(dispatcher *d) { return __sync_bool_compare_and_swap(&(d->route_refresh_pending), 0, 0); } /** * Returns the wall-clock time in milliseconds consumed by this dispatcher. */ inline size_t dispatch_get_ticks(dispatcher *self) { return __sync_add_and_fetch(&(self->ticks), 0); } /** * Returns the wall-clock time consumed since last call to this * function. */ inline size_t dispatch_get_ticks_sub(dispatcher *self) { size_t d = dispatch_get_ticks(self) - self->prevticks; self->prevticks += d; return d; } /** * Returns the wall-clock time in milliseconds consumed while sleeping * by this dispatcher. */ inline size_t dispatch_get_sleeps(dispatcher *self) { return __sync_add_and_fetch(&(self->sleeps), 0); } /** * Returns the wall-clock time consumed while sleeping since last call * to this function. */ inline size_t dispatch_get_sleeps_sub(dispatcher *self) { size_t d = dispatch_get_sleeps(self) - self->prevsleeps; self->prevsleeps += d; return d; } /** * Returns the number of metrics dispatched since start. */ inline size_t dispatch_get_metrics(dispatcher *self) { return __sync_add_and_fetch(&(self->metrics), 0); } /** * Returns the number of metrics dispatched since last call to this * function. */ inline size_t dispatch_get_metrics_sub(dispatcher *self) { size_t d = dispatch_get_metrics(self) - self->prevmetrics; self->prevmetrics += d; return d; } /** * Returns the number of metrics that were explicitly or implicitly * blackholed since start. */ inline size_t dispatch_get_blackholes(dispatcher *self) { return __sync_add_and_fetch(&(self->blackholes), 0); } /** * Returns the number of metrics that were blackholed since last call to * this function. */ inline size_t dispatch_get_blackholes_sub(dispatcher *self) { size_t d = dispatch_get_blackholes(self) - self->prevblackholes; self->prevblackholes += d; return d; } /** * Returns the number of accepted connections thusfar. */ inline size_t dispatch_get_accepted_connections(void) { return __sync_add_and_fetch(&(acceptedconnections), 0); } /** * Returns the number of closed connections thusfar. */ inline size_t dispatch_get_closed_connections(void) { return __sync_add_and_fetch(&(closedconnections), 0); } carbon-c-relay-2.5/dispatcher.h000066400000000000000000000035231303466152100164610ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef DISPATCHER_H #define DISPATCHER_H 1 #include #include "router.h" typedef struct _dispatcher dispatcher; void dispatch_check_rlimit_and_warn(void); int dispatch_addlistener(int sock); int dispatch_addlistener_udp(int sock); void dispatch_removelistener(int sock); int dispatch_addconnection(int sock); int dispatch_addconnection_aggr(int sock); void dispatch_set_bufsize(unsigned int sockbufsize); dispatcher *dispatch_new_listener(void); dispatcher *dispatch_new_connection(router *r, char *allowed_chars); void dispatch_stop(dispatcher *d); void dispatch_shutdown(dispatcher *d); void dispatch_free(dispatcher *d); size_t dispatch_get_ticks(dispatcher *self); size_t dispatch_get_metrics(dispatcher *self); size_t dispatch_get_blackholes(dispatcher *self); size_t dispatch_get_sleeps(dispatcher *self); size_t dispatch_get_ticks_sub(dispatcher *self); size_t dispatch_get_metrics_sub(dispatcher *self); size_t dispatch_get_blackholes_sub(dispatcher *self); size_t dispatch_get_sleeps_sub(dispatcher *self); size_t dispatch_get_accepted_connections(void); size_t dispatch_get_closed_connections(void); void dispatch_hold(dispatcher *d); void dispatch_schedulereload(dispatcher *d, router *r); char dispatch_reloadcomplete(dispatcher *d); #endif carbon-c-relay-2.5/fnv1a.h000066400000000000000000000025131303466152100153440ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define FNV1A_32_OFFSET 2166136261UL #define FNV1A_32_PRIME 16777619 /** * 32-bits unsigned FNV1a returning into hash, using p to as variable to * walk over metric up to firstspace */ #define fnv1a_32(hash, p, metric, firstspace) \ hash = FNV1A_32_OFFSET; \ for (p = metric; p < firstspace; p++) \ hash = (hash ^ (unsigned int)*p) * FNV1A_32_PRIME; #define FNV1A_64_OFFSET 14695981039346656037ULL #define FNV1A_64_PRIME 1099511628211UL /** * 64-bits unsigned FNV1a returning into hash, using p to as variable to * walk over metric up to firstspace */ #define fnv1a_64(hash, p, metric, firstspace) \ hash = FNV1A_64_OFFSET; \ for (p = metric; p < firstspace; p++) \ hash = (hash ^ (unsigned long long int)*p) * FNV1A_64_PRIME; carbon-c-relay-2.5/issues/000077500000000000000000000000001303466152100154725ustar00rootroot00000000000000carbon-c-relay-2.5/issues/distributiontest.c000066400000000000000000000074251303466152100212650ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* This is a clumpsy program to test the distribution of a certain input * set of metrics. See also: * https://github.com/graphite-project/carbon/issues/485 * * compile using something like this: * clang -o distributiontest -I. issues/distributiontest.c consistent-hash.c \ * server.c queue.c md5.c dispatcher.c router.c aggregator.c -pthread -lm */ #include #include #include #include #include #include #include "consistent-hash.h" #include "router.h" #include "server.h" #include "relay.h" #include "md5.h" #define SRVCNT 8 #define REPLCNT 2 unsigned char mode = 0; int relaylog(enum logdst dest, const char *fmt, ...) { (void) dest; (void) fmt; return 0; } int main(int argc, char *argv[]) { FILE *f; int i, j; ch_ring *r; char buf[METRIC_BUFSIZ]; server *s[SRVCNT]; size_t scnt[SRVCNT]; size_t metrics; time_t start, stop; size_t min, max; double mean, stddev; if (argc < 2) { fprintf(stderr, "need file argument\n"); return 1; } if (strcmp(argv[1], "-") == 0) { f = stdin; } else { f = fopen(argv[1], "r"); } if (f == NULL) { fprintf(stderr, "failed to open '%s': %s\n", argv[1], strerror(errno)); return 1; } /* build hash-ring */ r = ch_new(JUMP_FNV1a); /* or CARBON, FNV1a */ for (i = 0; i < SRVCNT; i++) { char ip[24]; unsigned char md5[MD5_DIGEST_LENGTH]; char md5sum[MD5_DIGEST_LENGTH * 2 + 1]; snprintf(ip, sizeof(ip), "192.168.%d.%d", i, 10 + (i * 2)); s[i] = server_new( ip, 2003, CON_TCP, NULL, 1024, 128, 4, 800 ); MD5((unsigned char *)ip, strlen(ip), md5); for (j = 0; j < MD5_DIGEST_LENGTH; j++) { snprintf(md5sum + (j * 2), 3, "%02x", md5[j]); } /* to use instance, enable this */ if (1 == 0) server_set_instance(s[i], md5sum); r = ch_addnode(r, s[i]); } /* process input */ memset(scnt, 0, sizeof(size_t) * SRVCNT); metrics = 0; start = time(NULL); while (fgets(buf, METRIC_BUFSIZ, f) != NULL) { destination dst[REPLCNT]; metrics++; ch_get_nodes( dst, r, REPLCNT, buf, buf + strlen(buf) - 1 ); for (i = 0; i < REPLCNT; i++) { free((void *)dst[i].metric); for (j = 0; j < SRVCNT; j++) { if (s[j] == dst[i].dest) { scnt[j]++; break; } } } } stop = time(NULL); mean = 0.0; stddev = 0.0; if (stop == start) { stop++; } printf("total metrics processed: %zd, time spent: ~%ds (~%d/s)\n", metrics, (int)(stop - start), (int)(metrics / (stop - start))); printf("replication count: %d, server count: %d\n", REPLCNT, SRVCNT); printf("server distribution:\n"); for (i = 0; i < SRVCNT; i++) { printf("- server %15s: %6zd (%.2f%%)\n", server_ip(s[i]), scnt[i], ((double)scnt[i] * 100.0) / (double)metrics); if (i == 0) { min = max = scnt[i]; } else { if (scnt[i] < min) min = scnt[i]; if (scnt[i] > max) max = scnt[i]; } mean += (double)scnt[i]; } mean /= (double)SRVCNT; for (i = 0; i < SRVCNT; i++) { stddev += pow((double)scnt[i] - mean, 2); } stddev = sqrt(stddev / (double)SRVCNT); printf("band: %zd - %zd (diff %zd, %.1f%%), mean: %.2f, stddev: %.2f\n", min, max, max - min, (double)(max-min) * 100.0 / (mean * SRVCNT), mean, stddev); } carbon-c-relay-2.5/issues/issue10.conf000066400000000000000000000020531303466152100176320ustar00rootroot00000000000000rewrite ^servers\.(cloud|bc|cwwtf|telhc)\.(int|test|stage|live|eng|green|mgmt)\.([a-zA-Z]+)([0-9]+) into servers.\1.\2.\3.\3\4 ; #remove derive, objects and gauges, etc. in keys (unless for applications (statsd)) # regex implementation doesn't understand this one #rewrite ^(.*)\.(? 1: arg = sys.argv[1] if arg.isdigit(): delay = int(arg) else: sys.stderr.write("Ignoring non-integer argument. Using default: %ss\n" % delay) sock = socket.socket() try: sock.connect( (CARBON_SERVER, CARBON_PORT) ) except socket.error: raise SystemExit("Couldn't connect to %(server)s on port %(port)d, is carbon-cache.py running?" % { 'server':CARBON_SERVER, 'port':CARBON_PORT }) try: run(sock, delay) except KeyboardInterrupt: sys.stderr.write("\nExiting on CTRL-c\n") sys.exit(0) if __name__ == "__main__": main() carbon-c-relay-2.5/issues/issue60.conf000066400000000000000000000007521303466152100176430ustar00rootroot00000000000000match ^metrics.all.* send to blackhole stop ; aggregate metrics.*.api.ac\.([^.]+)\.([^.]+)\.([^.]+)\.([^.]+)\.([^.]+)\.count every 10 seconds expire after 50 seconds compute sum write to metrics.all.api.ac.\1.\2.\3.\4.\5.count compute sum write to metrics.all.api.ac.\1.\2.all.\4.\5.count compute sum write to metrics.all.api.ac.\1.\2.\3.all.\5.count compute sum write to metrics.all.api.ac.\1.\2.all.all.\5.count ; carbon-c-relay-2.5/issues/issue82.conf000066400000000000000000000007751303466152100176540ustar00rootroot00000000000000cluster local_carbon carbon_ch 127.0.0.1:2013=a ; match * send to local_carbon stop ; aggregate ^sys\.dc[0-9].(somehost-[0-9]+)\.([^.]+)\.mysql\.replication_delay every 10 seconds expire after 35 seconds compute sum write to mysql.host.\1.replication_delay compute sum write to mysql.host.all.replication_delay compute sum write to mysql.cluster.\2.replication_delay compute sum write to mysql.cluster.all.replication_delay ; carbon-c-relay-2.5/issues/issue94-2.conf000066400000000000000000000004331303466152100200050ustar00rootroot00000000000000cluster graphite forward 127.0.0.1:2004 ; aggregate ^sys\.somemetric every 60 seconds expire after 75 seconds timestamp at end of bucket compute sum write to sys.somemetric send to graphite stop ; match * send to graphite ; carbon-c-relay-2.5/issues/issue94-3.conf000066400000000000000000000025301303466152100200060ustar00rootroot00000000000000############################################################################### # # Define clusters that we will manage. # ############################################################################### cluster local forward 127.0.0.1:2004 ; ############################################################################### # # Define any rewrite rules you might want. # ############################################################################### # carbon-c-relay metrics should go under system. rewrite ^carbon\.(.*) into system.carbon.\1 ; ############################################################################### # # Define Aggregations we want to use. # ############################################################################### #access-logs page counts aggregate ^app\.([^.]+)\.dc\.([^.]+)\.env\.([^.]+)\.store\.([^.]+)\.pages\.([^.]+)\.count every 60 seconds expire after 90 seconds compute sum write to aggregated_app.\1.dc.\2.env.\3.store.\4.pages.\5.count send to local ; ############################################################################### # # Send carbon metrics home. If you want another metric to be sent do it here. # ############################################################################### match ^system\.carbon\.(.*) send to local stop ; carbon-c-relay-2.5/issues/issue94-4.conf000066400000000000000000000004641303466152100200130ustar00rootroot00000000000000cluster reagg fnv1a_ch 127.0.0.1:2004=a 127.0.0.2:2004=b 127.0.0.3:2004=c 127.0.0.4:2004=d ; cluster b forward 127.0.0.1:2005; aggregate ^(.+).([^.]+).([^.]+)$ every 10 seconds expire after 30 seconds timestamp at end of bucket compute sum write to agg.\1.\2.sumall.hosts send to reagg b stop ; carbon-c-relay-2.5/issues/issue94.conf000066400000000000000000000011271303466152100176470ustar00rootroot00000000000000cluster aggregator-post failover 127.0.0.1:2004 127.0.0.1:2005 ; rewrite ^carbon\.(.*) into system.carbon.\1 ; aggregate ^app\.([^.]+)\.dc\.([^.]+)\.env\.([^.]+)\.store\.([^.]+)\.pages\.([^.]+)\.count every 60 seconds expire after 95 seconds timestamp at end of bucket compute sum write to aggregated_app.\1.dc.\2.env.\3.store.\4.pages.\5.count ; match ^system\.carbon\.(.*) send to aggregator-post stop ; match ^aggregated_(.*) send to aggregator-post stop ; match * send to blackhole stop ; carbon-c-relay-2.5/issues/routertest.c000066400000000000000000000032571303466152100200650ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* Simple example of how to use the routing functionality of the relay * https://github.com/grobian/carbon-c-relay/issues/192 * * compile with something like: * gcc -o routertest routertest.c router.c server.c consistent-hash.c \ * md5.c queue.c dispatcher.c aggregator.c -lnsl -lsocket -lm */ #include #include #include #include "relay.h" #include "server.h" #include "router.h" unsigned char mode = 0; relaylog(enum logdst dest, const char *fmt, ...) { return 0; } #define DESTS_SIZE 32 int main() { size_t len = 0; size_t i; destination dests[DESTS_SIZE]; char *metric; char *firstspace; server *s; router *rtr; if ((rtr = router_readconfig(NULL, "test.conf", 100, 10, 1, 100)) == NULL) exit(1); metric = strdup("bla.bla.bla"); firstspace = metric + strlen(metric); (void)router_route(rtr, dests, &len, DESTS_SIZE, "127.0.0.1", metric, firstspace); for (i = 0; i < len; i++) { free((char *)dests[i].metric); s = dests[i].dest; printf("%zd: %s:%d=%s\n", i, server_ip(s), server_port(s), server_instance(s)); } return 0; } carbon-c-relay-2.5/md5.c000066400000000000000000000211111303466152100150040ustar00rootroot00000000000000/* * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. * MD5 Message-Digest Algorithm (RFC 1321). * * Homepage: * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 * * Author: * Alexander Peslyak, better known as Solar Designer * * This software was written by Alexander Peslyak in 2001. No copyright is * claimed, and the software is hereby placed in the public domain. * In case this attempt to disclaim copyright and place the software in the * public domain is deemed null and void, then the software is * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the * general public under the following terms: * * Redistribution and use in source and binary forms, with or without * modification, are permitted. * * There's ABSOLUTELY NO WARRANTY, express or implied. * * (This is a heavily cut-down "BSD license".) * * This differs from Colin Plumb's older public domain implementation in that * no exactly 32-bit integer data type is required (any 32-bit or wider * unsigned integer data type will do), there's no compile-time endianness * configuration, and the function prototypes match OpenSSL's. No code from * Colin Plumb's implementation has been reused; this comment merely compares * the properties of the two independent implementations. * * The primary goals of this implementation are portability and ease of use. * It is meant to be fast, but not as fast as possible. Some known * optimizations are not included to reduce source code size and avoid * compile-time configuration. */ #ifndef HAVE_OPENSSL #include #include "md5.h" /* * The basic MD5 functions. * * F and G are optimized compared to their RFC 1321 definitions for * architectures that lack an AND-NOT instruction, just like in Colin Plumb's * implementation. */ #define F(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) #define G(x, y, z) ((y) ^ ((z) & ((x) ^ (y)))) #define H(x, y, z) (((x) ^ (y)) ^ (z)) #define H2(x, y, z) ((x) ^ ((y) ^ (z))) #define I(x, y, z) ((y) ^ ((x) | ~(z))) /* * The MD5 transformation for all four rounds. */ #define STEP(f, a, b, c, d, x, t, s) \ (a) += f((b), (c), (d)) + (x) + (t); \ (a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \ (a) += (b); /* * SET reads 4 input bytes in little-endian byte order and stores them * in a properly aligned word in host byte order. * * The check for little-endian architectures that tolerate unaligned * memory accesses is just an optimization. Nothing will break if it * doesn't work. */ #if defined(__i386__) || defined(__x86_64__) || defined(__vax__) #define SET(n) \ (*(MD5_u32plus *)&ptr[(n) * 4]) #define GET(n) \ SET(n) #else #define SET(n) \ (ctx->block[(n)] = \ (MD5_u32plus)ptr[(n) * 4] | \ ((MD5_u32plus)ptr[(n) * 4 + 1] << 8) | \ ((MD5_u32plus)ptr[(n) * 4 + 2] << 16) | \ ((MD5_u32plus)ptr[(n) * 4 + 3] << 24)) #define GET(n) \ (ctx->block[(n)]) #endif /* * This processes one or more 64-byte data blocks, but does NOT update * the bit counters. There are no alignment requirements. */ static const void *body(MD5_CTX *ctx, const void *data, unsigned long size) { const unsigned char *ptr; MD5_u32plus a, b, c, d; MD5_u32plus saved_a, saved_b, saved_c, saved_d; ptr = (const unsigned char *)data; a = ctx->a; b = ctx->b; c = ctx->c; d = ctx->d; do { saved_a = a; saved_b = b; saved_c = c; saved_d = d; /* Round 1 */ STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7) STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12) STEP(F, c, d, a, b, SET(2), 0x242070db, 17) STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22) STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7) STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12) STEP(F, c, d, a, b, SET(6), 0xa8304613, 17) STEP(F, b, c, d, a, SET(7), 0xfd469501, 22) STEP(F, a, b, c, d, SET(8), 0x698098d8, 7) STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12) STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17) STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22) STEP(F, a, b, c, d, SET(12), 0x6b901122, 7) STEP(F, d, a, b, c, SET(13), 0xfd987193, 12) STEP(F, c, d, a, b, SET(14), 0xa679438e, 17) STEP(F, b, c, d, a, SET(15), 0x49b40821, 22) /* Round 2 */ STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5) STEP(G, d, a, b, c, GET(6), 0xc040b340, 9) STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14) STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20) STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5) STEP(G, d, a, b, c, GET(10), 0x02441453, 9) STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14) STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20) STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5) STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9) STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14) STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20) STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5) STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9) STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14) STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20) /* Round 3 */ STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4) STEP(H2, d, a, b, c, GET(8), 0x8771f681, 11) STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16) STEP(H2, b, c, d, a, GET(14), 0xfde5380c, 23) STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4) STEP(H2, d, a, b, c, GET(4), 0x4bdecfa9, 11) STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16) STEP(H2, b, c, d, a, GET(10), 0xbebfbc70, 23) STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4) STEP(H2, d, a, b, c, GET(0), 0xeaa127fa, 11) STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16) STEP(H2, b, c, d, a, GET(6), 0x04881d05, 23) STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4) STEP(H2, d, a, b, c, GET(12), 0xe6db99e5, 11) STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16) STEP(H2, b, c, d, a, GET(2), 0xc4ac5665, 23) /* Round 4 */ STEP(I, a, b, c, d, GET(0), 0xf4292244, 6) STEP(I, d, a, b, c, GET(7), 0x432aff97, 10) STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15) STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21) STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6) STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10) STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15) STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21) STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6) STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10) STEP(I, c, d, a, b, GET(6), 0xa3014314, 15) STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21) STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6) STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10) STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15) STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21) a += saved_a; b += saved_b; c += saved_c; d += saved_d; ptr += 64; } while (size -= 64); ctx->a = a; ctx->b = b; ctx->c = c; ctx->d = d; return ptr; } void MD5_Init(MD5_CTX *ctx) { ctx->a = 0x67452301; ctx->b = 0xefcdab89; ctx->c = 0x98badcfe; ctx->d = 0x10325476; ctx->lo = 0; ctx->hi = 0; } void MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size) { MD5_u32plus saved_lo; unsigned long used, available; saved_lo = ctx->lo; if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo) ctx->hi++; ctx->hi += size >> 29; used = saved_lo & 0x3f; if (used) { available = 64 - used; if (size < available) { memcpy(&ctx->buffer[used], data, size); return; } memcpy(&ctx->buffer[used], data, available); data = (const unsigned char *)data + available; size -= available; body(ctx, ctx->buffer, 64); } if (size >= 64) { data = body(ctx, data, size & ~(unsigned long)0x3f); size &= 0x3f; } memcpy(ctx->buffer, data, size); } void MD5_Final(unsigned char *result, MD5_CTX *ctx) { unsigned long used, available; used = ctx->lo & 0x3f; ctx->buffer[used++] = 0x80; available = 64 - used; if (available < 8) { memset(&ctx->buffer[used], 0, available); body(ctx, ctx->buffer, 64); used = 0; available = 64; } memset(&ctx->buffer[used], 0, available - 8); ctx->lo <<= 3; ctx->buffer[56] = ctx->lo; ctx->buffer[57] = ctx->lo >> 8; ctx->buffer[58] = ctx->lo >> 16; ctx->buffer[59] = ctx->lo >> 24; ctx->buffer[60] = ctx->hi; ctx->buffer[61] = ctx->hi >> 8; ctx->buffer[62] = ctx->hi >> 16; ctx->buffer[63] = ctx->hi >> 24; body(ctx, ctx->buffer, 64); result[0] = ctx->a; result[1] = ctx->a >> 8; result[2] = ctx->a >> 16; result[3] = ctx->a >> 24; result[4] = ctx->b; result[5] = ctx->b >> 8; result[6] = ctx->b >> 16; result[7] = ctx->b >> 24; result[8] = ctx->c; result[9] = ctx->c >> 8; result[10] = ctx->c >> 16; result[11] = ctx->c >> 24; result[12] = ctx->d; result[13] = ctx->d >> 8; result[14] = ctx->d >> 16; result[15] = ctx->d >> 24; memset(ctx, 0, sizeof(*ctx)); } unsigned char *MD5(const void *data, unsigned long size, unsigned char *result) { MD5_CTX ctx; MD5_Init(&ctx); MD5_Update(&ctx, data, size); MD5_Final(result, &ctx); return result; } #endif carbon-c-relay-2.5/md5.h000066400000000000000000000027701303466152100150230ustar00rootroot00000000000000/* * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. * MD5 Message-Digest Algorithm (RFC 1321). * * Homepage: * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 * * Author: * Alexander Peslyak, better known as Solar Designer * * This software was written by Alexander Peslyak in 2001. No copyright is * claimed, and the software is hereby placed in the public domain. * In case this attempt to disclaim copyright and place the software in the * public domain is deemed null and void, then the software is * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the * general public under the following terms: * * Redistribution and use in source and binary forms, with or without * modification, are permitted. * * There's ABSOLUTELY NO WARRANTY, express or implied. * * See md5.c for more information. */ #ifdef HAVE_OPENSSL #include #elif !defined(_MD5_H) #define _MD5_H #define MD5_DIGEST_LENGTH 16 /* Any 32-bit or wider unsigned integer data type will do */ typedef unsigned int MD5_u32plus; typedef struct { MD5_u32plus lo, hi; MD5_u32plus a, b, c, d; unsigned char buffer[64]; MD5_u32plus block[16]; } MD5_CTX; extern void MD5_Init(MD5_CTX *ctx); extern void MD5_Update(MD5_CTX *ctx, const void *data, unsigned long size); extern void MD5_Final(unsigned char *result, MD5_CTX *ctx); extern unsigned char *MD5(const void *data, unsigned long size, unsigned char *result); #endif carbon-c-relay-2.5/queue.c000066400000000000000000000117031303466152100154510ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include "queue.h" struct _queue { const char **queue; size_t end; size_t write; size_t read; size_t len; pthread_mutex_t lock; }; /** * Allocates a new queue structure with capacity to hold size elements. */ queue * queue_new(size_t size) { queue *ret = malloc(sizeof(queue)); if (ret == NULL) return NULL; ret->queue = malloc(sizeof(char *) * size); if (ret->queue == NULL) { free(ret); return NULL; } memset(ret->queue, 0, sizeof(char *) * size); ret->end = size; ret->read = ret->write = 0; ret->len = 0; pthread_mutex_init(&ret->lock, NULL); return ret; } /** * Frees up allocated resources in use by the queue. This doesn't take * into account any consumers at all. That is, the caller needs to * ensure noone is using the queue any more. */ void queue_destroy(queue *q) { const char *p; /* drain queue not to leak the memory consumed by pending metrics */ while ((p = queue_dequeue(q)) != NULL) free((char *)p); q->len = 0; pthread_mutex_destroy(&q->lock); free(q->queue); free(q); } /** * Enqueues the string pointed to by p at queue q. If the queue is * full, the oldest entry is dropped. For this reason, enqueuing will * never fail. This function assumes the pointer p is a copy for this * queue, that is returned on dequeue, or freed when dropped. */ void queue_enqueue(queue *q, const char *p) { /* queue normal: * |=====-----------------------------| 4 * ^ ^ * r w * queue wrap: * |===---------------------------====| 6 * ^ ^ * w r * queue full * |==================================| 23 * ^ * w+r */ pthread_mutex_lock(&q->lock); if (q->len == q->end) { if (q->read == q->end) q->read = 0; free((char *)(q->queue[q->read])); q->read++; q->len--; } if (q->write == q->end) q->write = 0; q->queue[q->write] = p; q->write++; q->len++; pthread_mutex_unlock(&q->lock); } /** * Returns the oldest entry in the queue. If there are no entries, NULL * is returned. The caller should free the returned string. */ const char * queue_dequeue(queue *q) { const char *ret; pthread_mutex_lock(&q->lock); if (q->len == 0) { pthread_mutex_unlock(&q->lock); return NULL; } if (q->read == q->end) q->read = 0; ret = q->queue[q->read++]; q->len--; pthread_mutex_unlock(&q->lock); return ret; } /** * Returns at most len elements from the queue. Attempts to use a * single lock to read a vector of elements from the queue to minimise * effects of locking. Returns the number of elements stored in ret. * The caller is responsible for freeing elements from ret, as well as * making sure it is large enough to store len elements. */ size_t queue_dequeue_vector(const char **ret, queue *q, size_t len) { size_t i; pthread_mutex_lock(&q->lock); if (q->len == 0) { pthread_mutex_unlock(&q->lock); return 0; } if (len > q->len) len = q->len; for (i = 0; i < len; i++) { if (q->read == q->end) q->read = 0; ret[i] = q->queue[q->read++]; } q->len -= len; pthread_mutex_unlock(&q->lock); return len; } /** * Puts the entry p at the front of the queue, instead of the end, if * there is space available in the queue. Returns 0 when no space is * available, non-zero otherwise. Like queue_enqueue, * queue_putback assumes pointer p points to a private copy for the * queue. */ char queue_putback(queue *q, const char *p) { pthread_mutex_lock(&q->lock); if (q->len == q->end) { pthread_mutex_unlock(&q->lock); return 0; } if (q->read == 0) q->read = q->end; q->read--; q->queue[q->read] = p; q->len++; pthread_mutex_unlock(&q->lock); return 1; } /** * Returns the (approximate) size of entries waiting to be read in the * queue. The returned value cannot be taken accurate with multiple * readers/writers concurrently in action. Hence it can only be seen as * mere hint about the state of the queue. */ inline size_t queue_len(queue *q) { size_t len; pthread_mutex_lock(&q->lock); len = q->len; pthread_mutex_unlock(&q->lock); return len; } /** * Returns the (approximate) size of free entries in the queue. The * same conditions as for queue_len apply. */ inline size_t queue_free(queue *q) { return q->end - queue_len(q); } /** * Returns the size of the queue. */ inline size_t queue_size(queue *q) { return q->end; } carbon-c-relay-2.5/queue.h000066400000000000000000000020221303466152100154500ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef QUEUE_H #define QUEUE_H 1 #include typedef struct _queue queue; queue* queue_new(size_t size); void queue_destroy(queue *q); void queue_enqueue(queue *q, const char *p); const char *queue_dequeue(queue *q); size_t queue_dequeue_vector(const char **ret, queue *q, size_t len); char queue_putback(queue *q, const char *p); size_t queue_len(queue *q); size_t queue_free(queue *q); size_t queue_size(queue *q); #endif carbon-c-relay-2.5/receptor.c000066400000000000000000000127421303466152100161540ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include #include "relay.h" #ifndef TMPDIR # define TMPDIR "/tmp" #endif #define SOCKFILE ".s.carbon-c-relay" /** * Opens up listener sockets. Returns the socket fds in ret, and * updates retlen. If opening sockets failed, -1 is returned. The * caller should ensure retlen is at least 1, and ret should be an array * large enough to hold it. */ int bindlisten( int ret_stream[], int *retlen_stream, int ret_dgram[], int *retlen_dgram, const char *interface, unsigned short port, unsigned int backlog) { int sock; int optval; struct timeval tv; struct addrinfo hint; struct addrinfo *res, *resw; char buf[128]; char saddr[INET6_ADDRSTRLEN]; int err; int binderr = 0; int curlen_stream = 0; int curlen_dgram = 0; int socktypes[] = {SOCK_STREAM, SOCK_DGRAM, 0}; int *socktype = socktypes; tv.tv_sec = 0; tv.tv_usec = 500 * 1000; for (; *socktype != 0 && binderr == 0; socktype++) { memset(&hint, 0, sizeof(hint)); hint.ai_family = PF_UNSPEC; hint.ai_socktype = *socktype; hint.ai_protocol = 0; hint.ai_flags = AI_ADDRCONFIG | AI_NUMERICSERV | AI_PASSIVE; snprintf(buf, sizeof(buf), "%u", port); if ((err = getaddrinfo(interface, buf, &hint, &res)) != 0) { logerr("getaddrinfo(%s, %s, ...) failed: %s\n", interface == NULL ? "NULL" : interface, buf, gai_strerror(err)); return -1; } for (resw = res; resw != NULL; resw = resw->ai_next) { if (resw->ai_family != PF_INET && resw->ai_family != PF_INET6) continue; if (resw->ai_protocol != IPPROTO_TCP && resw->ai_protocol != IPPROTO_UDP) continue; if ((sock = socket(resw->ai_family, resw->ai_socktype, resw->ai_protocol)) < 0) { while (curlen_dgram > 0) close(ret_dgram[--curlen_dgram]); while (curlen_stream > 0) close(ret_stream[--curlen_stream]); binderr = 1; break; } (void) setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); optval = 1; /* allow takeover */ (void) setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); if (resw->ai_family == PF_INET6) { optval = 1; (void) setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &optval, sizeof(optval)); } snprintf(saddr, sizeof(saddr), "(unknown)"); switch (resw->ai_family) { case PF_INET: inet_ntop(resw->ai_family, &((struct sockaddr_in *)resw->ai_addr)->sin_addr, saddr, sizeof(saddr)); break; case PF_INET6: inet_ntop(resw->ai_family, &((struct sockaddr_in6 *)resw->ai_addr)->sin6_addr, saddr, sizeof(saddr)); break; } if (bind(sock, resw->ai_addr, resw->ai_addrlen) < 0) { logerr("failed to bind on %s%d %s port %s\n", resw->ai_protocol == IPPROTO_TCP ? "tcp" : "udp", resw->ai_family == PF_INET6 ? 6 : 4, saddr, buf); close(sock); while (curlen_dgram > 0) close(ret_dgram[--curlen_dgram]); while (curlen_stream > 0) close(ret_stream[--curlen_stream]); binderr = 1; break; } if (resw->ai_protocol == IPPROTO_TCP) { if (listen(sock, backlog) < 0) { close(sock); while (curlen_dgram > 0) close(ret_dgram[--curlen_dgram]); while (curlen_stream > 0) close(ret_stream[--curlen_stream]); binderr = 1; break; } if (curlen_stream < *retlen_stream) { logout("listening on tcp%d %s port %s\n", resw->ai_family == PF_INET6 ? 6 : 4, saddr, buf); ret_stream[curlen_stream++] = sock; } } else { if (curlen_dgram < *retlen_dgram) { logout("listening on udp%d %s port %s\n", resw->ai_family == PF_INET6 ? 6 : 4, saddr, buf); ret_dgram[curlen_dgram++] = sock; } } } freeaddrinfo(res); } if (curlen_stream + curlen_dgram == 0) return -1; /* fake loop to simplify breakout below */ while (curlen_stream < *retlen_stream) { struct sockaddr_un server; #ifndef PF_LOCAL # define PF_LOCAL PF_UNIX #endif if ((sock = socket(PF_LOCAL, SOCK_STREAM, 0)) < 0) break; snprintf(buf, sizeof(buf), "%s/%s.%u", TMPDIR, SOCKFILE, port); memset(&server, 0, sizeof(struct sockaddr_un)); server.sun_family = PF_LOCAL; strncpy(server.sun_path, buf, sizeof(server.sun_path) - 1); unlink(buf); /* avoid address already in use */ if (bind(sock, (struct sockaddr *)&server, sizeof(struct sockaddr_un)) < 0) { logerr("failed to bind for %s: %s\n", buf, strerror(errno)); close(sock); break; } if (listen(sock, backlog) < 0) { close(sock); break; } logout("listening on UNIX socket %s\n", buf); ret_stream[curlen_stream++] = sock; break; } *retlen_stream = curlen_stream; *retlen_dgram = curlen_dgram; return 0; } void destroy_usock(unsigned short port) { char buf[512]; snprintf(buf, sizeof(buf), "%s/%s.%u", TMPDIR, SOCKFILE, port); unlink(buf); } carbon-c-relay-2.5/receptor.h000066400000000000000000000015241303466152100161550ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef RECEPTOR_H #define RECEPTOR_H 1 int bindlisten(int ret_stream[], int *retlen_stream, int ret_dgram[], int *retlen_dgram, const char *interface, unsigned short port, unsigned int backlog); void destroy_usock(unsigned short port); #endif carbon-c-relay-2.5/relay.c000066400000000000000000000604651303466152100154520ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #if defined(__MACH__) || defined(BSD) # include # include #endif #include "relay.h" #include "server.h" #include "router.h" #include "receptor.h" #include "dispatcher.h" #include "aggregator.h" #include "collector.h" unsigned char keep_running = 1; int pending_signal = -1; char relay_hostname[256]; unsigned char mode = 0; static char *config = NULL; static int batchsize = 2500; static int queuesize = 25000; static int maxstalls = 4; static unsigned short iotimeout = 600; static int sockbufsize = 0; static dispatcher **workers = NULL; static char workercnt = 0; static router *rtr = NULL; static server *internal_submission = NULL; static char *relay_logfile = NULL; static FILE *relay_stdout = NULL; static FILE *relay_stderr = NULL; static char relay_can_log = 0; /** * Writes to the setup output stream, prefixed with a timestamp, and if * the stream is not going to stdout or stderr, prefixed with MSG or * ERR. */ int relaylog(enum logdst dest, const char *fmt, ...) { va_list ap; char prefix[64]; size_t len; time_t now; struct tm tm_now; FILE *dst = NULL; char console = 0; int ret; /* briefly stall if we're swapping fds */ while (!__sync_add_and_fetch(&relay_can_log, 0)) usleep((100 + (rand() % 200)) * 1000); /* 100ms - 300ms */ switch (dest) { case LOGOUT: dst = relay_stdout; if (dst == stdout) console = 1; break; case LOGERR: dst = relay_stderr; if (dst == stderr) console = 1; break; } assert(dst != NULL); time(&now); localtime_r(&now, &tm_now); len = strftime(prefix, sizeof(prefix), "[%Y-%m-%d %H:%M:%S]", &tm_now); if (!console) (void)snprintf(prefix + len, sizeof(prefix) - len, " (%s)", dest == LOGOUT ? "MSG" : "ERR"); fprintf(dst, "%s ", prefix); va_start(ap, fmt); ret = vfprintf(dst, fmt, ap); fflush(dst); va_end(ap); return ret; } static void sig_handler(int sig) { /* a thunder of signals will be ignored for as long as a pending * signal was set, I can live with that, feels like a bad practice * to bombard a process like that */ (void)__sync_bool_compare_and_swap(&pending_signal, -1, sig); } static void do_reload(void) { router *newrtr; aggregator *newaggrs; int id; FILE *newfd; size_t numaggregators; if (relay_stderr != stderr) { /* try to re-open the file first, so we can still try and say * something if that fails */ if ((newfd = fopen(relay_logfile, "a")) == NULL) { logerr("not reopening logfiles: can't open '%s': %s\n", relay_logfile, strerror(errno)); } else { logout("closing logfile\n"); __sync_and_and_fetch(&relay_can_log, 0); /* lame race avoidance for relaylog() usage */ usleep((100 + (rand() % 200)) * 1000); /* 100ms - 300ms */ fclose(relay_stderr); relay_stdout = newfd; relay_stderr = newfd; __sync_bool_compare_and_swap(&relay_can_log, 0, 1); logout("reopening logfile\n"); } } logout("reloading config from '%s'\n", config); if ((newrtr = router_readconfig(NULL, config, queuesize, batchsize, maxstalls, iotimeout, sockbufsize)) == NULL) { logerr("failed to read configuration '%s', aborting reload\n", config); return; } router_optimise(newrtr); /* compare the configs first, if there is no diff, then * just refrain from reloading anything */ if (!router_printdiffs(rtr, newrtr, relay_stdout)) { /* no changes, shortcut */ logout("no config changes found\n"); router_free(newrtr); logout("SIGHUP handler complete\n"); return; } logout("reloading collector\n"); collector_schedulereload(newrtr); while (!collector_reloadcomplete()) usleep((100 + (rand() % 200)) * 1000); /* 100ms - 300ms */ /* During aggregator final expiry, the dispatchers should not put * new metrics, so we have to temporarily stop them processing * connections. However, we still need to handle aggregations that * we will stop next, so this call results in the dispatchers only * dealing with connections from aggregators. Doing this, also * means that we can reload the config atomicly, which disrupts the * service seemingly for a bit, but results in less confusing output * in the end. */ logout("interrupting workers\n"); for (id = 1; id < 1 + workercnt; id++) dispatch_hold(workers[id]); numaggregators = aggregator_numaggregators(router_getaggregators(rtr)); if (numaggregators > 0) { logout("expiring aggregations\n"); aggregator_stop(); /* frees aggrs */ /* Now the aggregator has written everything to the * dispatchers, which will hand over to the servers with proper * metric adjustments (stubs). */ } newaggrs = router_getaggregators(newrtr); numaggregators = aggregator_numaggregators(newaggrs); if (numaggregators > 0) { if (!aggregator_start(newaggrs)) { logerr("failed to start aggregator, aggregations will no " "longer produce output!\n"); } } /* Transplant the queues for the same servers, so we keep their * queues (potentially with data) around. To do so, we need to make * sure the servers aren't being operated on. */ router_shutdown(rtr); /* Because we know that ip:port:prot is unique in both sets, when we * find a match, we can just swap the queue. This comes with a * slight peculiarity, that if a server becomes part of an any_of * cluster (or failover), its pending queue can now be processed by * other servers. While this feels wrong, it can be extremely * useful in case existing data needs to be migrated to another * server, e.g. the config on purpose added a failover to offload a * known dead server before removing it. This is fairly specific * but in reality this happens to be desirable every once so often. */ router_transplant_queues(newrtr, rtr); /* Before we start the workers sending traffic, start up the * servers now queues are transplanted. */ router_start(newrtr); logout("reloading workers\n"); for (id = 1; id < 1 + workercnt; id++) dispatch_schedulereload(workers[id], newrtr); /* un-holds */ for (id = 1; id < 1 + workercnt; id++) { while (!dispatch_reloadcomplete(workers[id + 0])) usleep((100 + (rand() % 200)) * 1000); /* 100ms - 300ms */ } router_free(rtr); rtr = newrtr; logout("SIGHUP handler complete\n"); } static void handle_signal(int sig) { char *sign = "unknown signal"; switch (sig) { case SIGTERM: sign = "SIGTERM"; break; case SIGINT: sign = "SIGINT"; break; case SIGQUIT: sign = "SIGQUIT"; break; case SIGHUP: sign = "SIGHUP"; break; } if (keep_running) { logout("caught %s\n", sign); } else { /* also fine when SIGHUP */ logerr("caught %s while already shutting down, " "forcing exit!\n", sign); exit(1); } if (sig == SIGHUP) { do_reload(); } else { keep_running = 0; } } static int get_cores(void) { int cpus = 5; #if defined(__MACH__) || defined(BSD) size_t len = sizeof(cpus); sysctlbyname("hw.ncpu", &cpus, &len, NULL, 0); #else /* Linux and Solaris */ cpus = sysconf(_SC_NPROCESSORS_ONLN); #endif return cpus; } static void do_version(void) { printf("carbon-c-relay v" VERSION " (" GIT_VERSION ")\n"); exit(0); } static void do_usage(char *name, int exitcode) { printf("Usage: %s -f [opts]\n", name); printf("\n"); printf("Options:\n"); printf(" -v print version and exit\n"); printf(" -f read for clusters and routes\n"); printf(" -p listen on for connections, defaults to 2003\n"); printf(" -i listen on for connections, defaults to all\n"); printf(" -l write output to , defaults to stdout/stderr\n"); printf(" -w use worker threads, defaults to %d\n", get_cores()); printf(" -b server send batch size, defaults to %d\n", batchsize); printf(" -q server queue size, defaults to %d\n", queuesize); printf(" -L server max stalls, defaults to %d\n", maxstalls); printf(" -S statistics sending interval in seconds, defaults to 60\n"); printf(" -B connection listen backlog, defaults to 32\n"); printf(" -T IO timeout in milliseconds for server connections, defaults to %d\n", iotimeout); printf(" -m send statistics like carbon-cache.py, e.g. not cumulative\n"); printf(" -c characters to allow next to [A-Za-z0-9], defaults to -_:#\n"); printf(" -d debug mode: currently writes statistics to log, prints hash\n" " ring contents and matching position in test mode (-t)\n"); printf(" -s submission mode: don't add any metrics to the stream like\n" " statistics, report drop counts and queue pressure to log\n"); printf(" -t config test mode: prints rule matches from input on stdin\n"); printf(" -H hostname: override hostname (used in statistics)\n"); printf(" -D daemonise: run in a background\n"); printf(" -P pidfile: write a pid to a specified pidfile\n"); exit(exitcode); } int main(int argc, char * const argv[]) { int stream_sock[] = {0, 0, 0}; /* tcp4, tcp6, UNIX */ int stream_socklen = sizeof(stream_sock) / sizeof(stream_sock[0]); int dgram_sock[] = {0, 0}; /* udp4, udp6 */ int dgram_socklen = sizeof(dgram_sock) / sizeof(dgram_sock[0]); char id; unsigned short listenport = 2003; unsigned int listenbacklog = 32; int ch; size_t numaggregators; char *listeninterface = NULL; char *allowed_chars = NULL; enum { SUB, CUM } smode = CUM; char *pidfile = NULL; FILE *pidfile_handle = NULL; aggregator *aggrs = NULL; if (gethostname(relay_hostname, sizeof(relay_hostname)) < 0) snprintf(relay_hostname, sizeof(relay_hostname), "127.0.0.1"); while ((ch = getopt(argc, argv, ":hvdmstf:i:l:p:w:b:q:L:S:T:c:H:B:U:DP:")) != -1) { switch (ch) { case 'v': do_version(); break; case 'd': /* secret support for -dd (just trace) and -ddd (debug * and trace) */ if (mode & MODE_DEBUG) { mode |= MODE_TRACE; mode &= ~MODE_DEBUG; } else { mode |= MODE_DEBUG; } break; case 'm': smode = SUB; break; case 's': mode |= MODE_SUBMISSION; break; case 't': mode |= MODE_TEST; break; case 'f': config = optarg; break; case 'i': listeninterface = optarg; break; case 'l': relay_logfile = optarg; break; case 'p': listenport = (unsigned short)atoi(optarg); if (listenport == 0) { fprintf(stderr, "error: port needs to be a number >0\n"); do_usage(argv[0], 1); } break; case 'w': workercnt = (char)atoi(optarg); if (workercnt <= 0) { fprintf(stderr, "error: workers needs to be a number >0\n"); do_usage(argv[0], 1); } break; case 'b': batchsize = atoi(optarg); if (batchsize <= 0) { fprintf(stderr, "error: batch size needs to be a number >0\n"); do_usage(argv[0], 1); } break; case 'q': queuesize = atoi(optarg); if (queuesize <= 0) { fprintf(stderr, "error: queue size needs to be a number >0\n"); do_usage(argv[0], 1); } break; case 'L': maxstalls = atoi(optarg); if (maxstalls < 0 || maxstalls >= (1 << SERVER_STALL_BITS)) { fprintf(stderr, "error: maximum stalls needs to be a number " "between 0 and %d\n", (1 << SERVER_STALL_BITS) - 1); do_usage(argv[0], 1); } break; case 'S': collector_interval = atoi(optarg); if (collector_interval <= 0) { fprintf(stderr, "error: sending interval needs to be " "a number >0\n"); do_usage(argv[0], 1); } break; case 'T': { int val = atoi(optarg); if (val <= 0) { fprintf(stderr, "error: server IO timeout needs " "to be a number >0\n"); do_usage(argv[0], 1); } else if (val >= 60000) { fprintf(stderr, "error: server IO timeout needs " "to be less than one minute\n"); do_usage(argv[0], 1); } iotimeout = (unsigned short)val; } break; case 'c': allowed_chars = optarg; break; case 'H': snprintf(relay_hostname, sizeof(relay_hostname), "%s", optarg); break; case 'B': { int val = atoi(optarg); if (val <= 0) { fprintf(stderr, "error: backlog needs to be a number >0\n"); do_usage(argv[0], 1); } listenbacklog = (unsigned int)val; } break; case 'U': { int val = atoi(optarg); socklen_t len = sizeof(sockbufsize); int sock; if (val <= 0) { fprintf(stderr, "error: bufsize needs to be a number >0\n"); do_usage(argv[0], 1); } sockbufsize = (unsigned int)val; /* test the size to see if the value is valid, since * this is OS dependant, it's the only way */ sock = socket(PF_INET, SOCK_STREAM, 0); if (sock == -1 || setsockopt(sock, SOL_SOCKET, SO_RCVBUF, &sockbufsize, sizeof(sockbufsize)) != 0) { fprintf(stderr, "failed to set socket bufsize: %s\n", strerror(errno)); } if (sock != -1 && getsockopt(sock, SOL_SOCKET, SO_RCVBUF, &sockbufsize, &len) != 0) { fprintf(stderr, "failed to get socket bufsize: %s\n", strerror(errno)); } if (len != sizeof(sockbufsize)) { fprintf(stderr, "getsockopt returned unexpected size: %u, " "expected %lu\n", len, sizeof(sockbufsize)); exit(1); } if (sockbufsize != (unsigned int)val) fprintf(stdout, "warning: OS rejected socket bufsize\n"); if (sock != -1) close(sock); } break; case 'D': mode |= MODE_DAEMON; break; case 'P': pidfile = optarg; break; case '?': case ':': do_usage(argv[0], 1); break; case 'h': default: do_usage(argv[0], 0); break; } } if (optind == 1 || config == NULL) do_usage(argv[0], 1); /* seed randomiser for dispatcher and aggregator "splay" */ srand(time(NULL)); if (workercnt == 0) workercnt = mode & MODE_SUBMISSION ? 2 : get_cores(); /* any_of failover maths need batchsize to be smaller than queuesize */ if (batchsize > queuesize) { fprintf(stderr, "error: batchsize must be smaller than queuesize\n"); exit(-1); } if (relay_logfile != NULL && (mode & MODE_TEST) == 0) { FILE *f = fopen(relay_logfile, "a"); if (f == NULL) { fprintf(stderr, "error: failed to open logfile '%s': %s\n", relay_logfile, strerror(errno)); exit(-1); } relay_stdout = f; relay_stderr = f; } else { relay_stdout = stdout; relay_stderr = stderr; } relay_can_log = 1; if (mode & MODE_DAEMON) { if (relay_logfile == NULL) { fprintf(stderr, "You must specify logfile if you want daemonisation!\n"); exit(-1); } if (mode & MODE_TEST) { fprintf(stderr, "You cannot use test mode if you want daemonisation!\n"); exit(-1); } } if (pidfile != NULL) { pidfile_handle = fopen(pidfile, "w"); if (pidfile_handle == NULL) { fprintf(stderr, "failed to open pidfile '%s': %s\n", pidfile, strerror(errno)); exit(-1); } } if (mode & MODE_DAEMON) { pid_t p; int fds[2]; if (pipe(fds) < 0) { fprintf(stderr, "failed to create pipe to talk to child process: %s\n", strerror(errno)); exit(1); } p = fork(); if (p < 0) { fprintf(stderr, "failed to fork intermediate process: %s\n", strerror(errno)); exit(1); } else if (p == 0) { char msg[1024]; /* child, for again so our intermediate process can die and * its child becomes an orphan to init */ p = fork(); if (p < 0) { snprintf(msg, sizeof(msg), "failed to fork daemon process: %s", strerror(errno)); /* fool compiler */ if (write(fds[1], msg, strlen(msg) + 1) > -2) exit(1); /* not that retcode matters */ } else if (p == 0) { /* child, this is the final daemon process */ if (setsid() < 0) { snprintf(msg, sizeof(msg), "failed to setsid(): %s", strerror(errno)); /* fool compiler */ if (write(fds[1], msg, strlen(msg) + 1) > -2) exit(1); /* not that retcode matters */ } /* close descriptors, we won't need them ever since * we're detached from the controlling terminal */ close(0); close(1); close(2); /* we're fine, flag our grandparent (= the original * process being called) it can terminate */ if (write(fds[1], "OK", 3) < -2) *msg = '\0'; /* unreachable dummy to fool compiler */ } else { /* parent, die */ exit(0); } } else { /* parent: wait for child to report success */ char msg[1024]; int len = read(fds[0], msg, sizeof(msg)); if (len < 0) { fprintf(stderr, "unable to read from child, " "assuming daemonisation failed\n"); exit(1); } else if (len == 0) { fprintf(stderr, "got unexpected EOF from child, " "assuming daemonisation failed\n"); exit(1); } else if (strncmp(msg, "OK", len) == 0) { /* child is happy, so can we */ exit(0); } else { if (len == 1024) len--; msg[len] = '\0'; fprintf(stderr, "%s\n", msg); exit(1); } } close(fds[0]); close(fds[1]); } if (pidfile_handle) { fprintf(pidfile_handle, "%d\n", getpid()); fclose(pidfile_handle); } logout("starting carbon-c-relay v%s (%s), pid=%d\n", VERSION, GIT_VERSION, getpid()); fprintf(relay_stdout, "configuration:\n"); fprintf(relay_stdout, " relay hostname = %s\n", relay_hostname); fprintf(relay_stdout, " listen port = %u\n", listenport); if (listeninterface != NULL) fprintf(relay_stdout, " listen interface = %s\n", listeninterface); fprintf(relay_stdout, " workers = %d\n", workercnt); fprintf(relay_stdout, " send batch size = %d\n", batchsize); fprintf(relay_stdout, " server queue size = %d\n", queuesize); fprintf(relay_stdout, " server max stalls = %d\n", maxstalls); fprintf(relay_stdout, " statistics submission interval = %ds\n", collector_interval); fprintf(relay_stdout, " listen backlog = %u\n", listenbacklog); if (sockbufsize > 0) fprintf(relay_stdout, " socket bufsize = %u\n", sockbufsize); fprintf(relay_stdout, " server connection IO timeout = %dms\n", iotimeout); if (allowed_chars != NULL) fprintf(relay_stdout, " extra allowed characters = %s\n", allowed_chars); if (mode & MODE_DEBUG) fprintf(relay_stdout, " debug = true\n"); #ifdef ENABLE_TRACE if (mode & MODE_TRACE) fprintf(relay_stdout, " trace = true\n"); #endif else if (mode & MODE_SUBMISSION) fprintf(relay_stdout, " submission = true\n"); else if (mode & MODE_DAEMON) fprintf(relay_stdout, " daemon = true\n"); fprintf(relay_stdout, " routes configuration = %s\n", config); fprintf(relay_stdout, "\n"); if ((rtr = router_readconfig(NULL, config, queuesize, batchsize, maxstalls, iotimeout, sockbufsize)) == NULL) { logerr("failed to read configuration '%s'\n", config); return 1; } router_optimise(rtr); aggrs = router_getaggregators(rtr); numaggregators = aggregator_numaggregators(aggrs); if (numaggregators > 10 && !(mode & MODE_DEBUG)) { fprintf(relay_stdout, "parsed configuration follows:\n" "(%zu aggregations with %zu computations omitted " "for brevity)\n", numaggregators, aggregator_numcomputes(aggrs)); router_printconfig(rtr, relay_stdout, PMODE_NORM); } else { fprintf(relay_stdout, "parsed configuration follows:\n"); router_printconfig(rtr, relay_stdout, PMODE_AGGR + (mode & MODE_DEBUG ? PMODE_DEBUG : PMODE_NORM)); } fprintf(relay_stdout, "\n"); /* shortcut for rule testing mode */ if (mode & MODE_TEST) { char metricbuf[METRIC_BUFSIZ]; char *p; fflush(relay_stdout); while (fgets(metricbuf, sizeof(metricbuf), stdin) != NULL) { if ((p = strchr(metricbuf, '\n')) != NULL) *p = '\0'; router_test(rtr, metricbuf); } exit(0); } if (signal(SIGINT, sig_handler) == SIG_ERR) { logerr("failed to create SIGINT handler: %s\n", strerror(errno)); return 1; } if (signal(SIGTERM, sig_handler) == SIG_ERR) { logerr("failed to create SIGTERM handler: %s\n", strerror(errno)); return 1; } if (signal(SIGQUIT, sig_handler) == SIG_ERR) { logerr("failed to create SIGQUIT handler: %s\n", strerror(errno)); return 1; } if (signal(SIGHUP, sig_handler) == SIG_ERR) { logerr("failed to create SIGHUP handler: %s\n", strerror(errno)); return 1; } if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) { logerr("failed to ignore SIGPIPE: %s\n", strerror(errno)); return 1; } workers = malloc(sizeof(dispatcher *) * (1 + workercnt + 1)); if (workers == NULL) { logerr("failed to allocate memory for workers\n"); return 1; } if (bindlisten(stream_sock, &stream_socklen, dgram_sock, &dgram_socklen, listeninterface, listenport, listenbacklog) < 0) { logerr("failed to bind on port %s:%d: %s\n", listeninterface == NULL ? "" : listeninterface, listenport, strerror(errno)); return -1; } dispatch_set_bufsize(sockbufsize); for (ch = 0; ch < stream_socklen; ch++) { if (dispatch_addlistener(stream_sock[ch]) != 0) { logerr("failed to add listener\n"); return -1; } } for (ch = 0; ch < dgram_socklen; ch++) { if (dispatch_addlistener_udp(dgram_sock[ch]) != 0) { logerr("failed to listen to datagram socket\n"); return -1; } } if ((workers[0] = dispatch_new_listener()) == NULL) logerr("failed to add listener\n"); if (allowed_chars == NULL) allowed_chars = "-_:#"; logout("starting %d workers\n", workercnt); for (id = 1; id < 1 + workercnt; id++) { workers[id + 0] = dispatch_new_connection(rtr, allowed_chars); if (workers[id + 0] == NULL) { logerr("failed to add worker %d\n", id); break; } } workers[id + 0] = NULL; if (id < 1 + workercnt) { logerr("shutting down due to errors\n"); keep_running = 0; } /* server used for delivering metrics produced inside the relay, * that is, the collector (statistics) */ if ((internal_submission = server_new( "internal", listenport, CON_PIPE, NULL, 3000, batchsize, maxstalls, iotimeout, sockbufsize)) == NULL) { logerr("failed to create internal submission queue, shutting down\n"); keep_running = 0; } if (internal_submission != NULL && server_start(internal_submission) != 0) { logerr("failed to start submission queue thread, shutting down\n"); server_free(internal_submission); internal_submission = NULL; keep_running = 0; } if (numaggregators > 0) { logout("starting aggregator\n"); if (!aggregator_start(aggrs)) { logerr("shutting down due to failure to start aggregator\n"); keep_running = 0; } } logout("starting statistics collector\n"); if (internal_submission != NULL) collector_start(&workers[1], rtr, internal_submission, smode == CUM); logout("starting servers\n"); if (router_start(rtr) != 0) { /* router_start logerrs itself */ keep_running = 0; } if (keep_running == 0) logout("startup sequence complete\n"); /* workers do the work, just wait */ while (keep_running) { sleep(1); if (pending_signal != -1) { handle_signal(pending_signal); pending_signal = -1; } } logout("shutting down...\n"); /* make sure we don't accept anything new anymore */ for (ch = 0; ch < stream_socklen; ch++) dispatch_removelistener(stream_sock[ch]); destroy_usock(listenport); logout("closed listeners for port %u\n", listenport); /* since workers will be freed, stop querying the structures */ collector_stop(); server_shutdown(internal_submission); server_free(internal_submission); logout("stopped collector\n"); if (numaggregators > 0) { aggregator_stop(); logout("stopped aggregator\n"); } /* give a little time for whatever the collector/aggregator wrote, * to be delivered by the dispatchers */ usleep(500 * 1000); /* 500ms */ /* make sure we don't write to our servers any more */ logout("stopped worker"); for (id = 0; id < 1 + workercnt; id++) dispatch_stop(workers[id + 0]); for (id = 0; id < 1 + workercnt; id++) { dispatch_shutdown(workers[id + 0]); dispatch_free(workers[id + 0]); fprintf(relay_stdout, " %d", id + 1); fflush(relay_stdout); } fprintf(relay_stdout, "\n"); fflush(relay_stdout); free(workers); router_shutdown(rtr); router_free(rtr); logout("stopped servers\n"); if (pidfile != NULL) unlink(pidfile); logout("stopped carbon-c-relay v%s (%s)\n", VERSION, GIT_VERSION); return 0; } carbon-c-relay-2.5/relay.h000066400000000000000000000025401303466152100154450ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef HAVE_RELAY_H #define HAVE_RELAY_H 1 #define VERSION "2.5" #define METRIC_BUFSIZ 8192 /* these are the various modes in which the relay runs */ #define MODE_DEBUG (1 << 0) #define MODE_SUBMISSION (1 << 1) #define MODE_TEST (1 << 2) #define MODE_DAEMON (1 << 3) #define MODE_TRACE (1 << 4) extern unsigned char mode; #ifdef ENABLE_TRACE #define tracef(...) if (mode & MODE_TRACE) fprintf(stdout, __VA_ARGS__) #else #define tracef(...) /* noop */ #endif typedef enum { CON_TCP, CON_UDP, CON_PIPE, CON_FILE } serv_ctype; extern char relay_hostname[]; enum logdst { LOGOUT, LOGERR }; int relaylog(enum logdst dest, const char *fmt, ...); #define logout(args...) relaylog(LOGOUT, args) #define logerr(args...) relaylog(LOGERR, args) #endif carbon-c-relay-2.5/router.c000066400000000000000000002567751303466152100156710ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fnv1a.h" #include "consistent-hash.h" #include "server.h" #include "queue.h" #include "aggregator.h" #include "relay.h" #include "router.h" enum clusttype { BLACKHOLE, /* /dev/null-like destination */ GROUP, /* pseudo type to create a matching tree */ AGGRSTUB, /* pseudo type to have stub matches for aggregation returns */ STATSTUB, /* pseudo type to have stub matches for collector returns */ VALIDATION, /* pseudo type to perform additional data validation */ FORWARD, FILELOG, /* like forward, write metric to file */ FILELOGIP, /* like forward, write ip metric to file */ CARBON_CH, /* original carbon-relay.py consistent-hash */ FNV1A_CH, /* FNV1a-based consistent-hash */ JUMP_CH, /* jump consistent hash with fnv1a input */ ANYOF, /* FNV1a-based hash, but with backup by others */ FAILOVER, /* ordered attempt delivery list */ AGGREGATION, REWRITE }; typedef struct _servers { server *server; int refcnt; struct _servers *next; } servers; typedef struct { unsigned char repl_factor; ch_ring *ring; servers *servers; } chashring; typedef struct { unsigned short count; server **servers; servers *list; } serverlist; typedef struct _validate { struct _route *rule; enum { VAL_LOG, VAL_DROP } action; } validate; typedef struct _cluster { char *name; enum clusttype type; union { chashring *ch; servers *forward; serverlist *anyof; aggregator *aggregation; struct _route *routes; char *replacement; struct _validate *validation; } members; struct _cluster *next; } cluster; typedef struct _destinations { cluster *cl; struct _destinations *next; } destinations; typedef struct _route { char *pattern; /* original regex input, used for printing only */ regex_t rule; /* regex on metric, only if type == REGEX */ size_t nmatch; /* number of match groups */ char *strmatch; /* string to search for if type not REGEX or MATCHALL */ destinations *dests; /* where matches should go */ char stop:1; /* whether to continue matching rules after this one */ enum { MATCHALL, /* the '*', don't do anything, just match everything */ REGEX, /* a regex match */ CONTAINS, /* find string occurrence */ STARTS_WITH, /* metric must start with string */ ENDS_WITH, /* metric must end with string */ MATCHES /* metric matches string exactly */ } matchtype; /* how to interpret the pattern */ struct _route *next; } route; struct _router { cluster *clusters; route *routes; aggregator *aggregators; servers *srvrs; char *collector_stub; struct _router_allocator { void *memory_region; void *nextp; size_t sz; struct _router_allocator *next; } *allocator; }; /* custom constant, meant to force regex mode matching */ #define REG_FORCE 01000000 /* catch string used for aggrgator destinations */ #define STUB_AGGR "_aggregator_stub_" /* catch string used for collector output */ #define STUB_STATS "_collector_stub_" /** * Free the allocators associated to this router. */ static void ra_free(router *rtr) { struct _router_allocator *ra; struct _router_allocator *ra_next; for (ra = rtr->allocator; ra != NULL; ra = ra_next) { free(ra->memory_region); ra_next = ra->next; free(ra); ra = NULL; } } /** * malloc() in one of the allocators for this router. If insufficient * memory is available in the allocator, a new one is allocated. If * that fails, NULL is returned, else a pointer that can be written to * up to sz bytes. */ static void * ra_malloc(router *rtr, size_t sz) { struct _router_allocator *ra; void *retp = NULL; size_t nsz; #define ra_alloc(RA, SZ) { \ assert(SZ >= 0); \ nsz = 256 * 1024; \ if (SZ > nsz) \ nsz = ((SZ / 1024) + 1) * 1024; \ RA = malloc(sizeof(struct _router_allocator)); \ if (RA == NULL) \ return NULL; \ RA->memory_region = malloc(sizeof(char) * nsz); \ if (RA->memory_region == NULL) { \ free(RA); \ RA = NULL; \ return NULL; \ } \ RA->nextp = RA->memory_region; \ RA->sz = nsz; \ RA->next = NULL; \ } if (rtr->allocator == NULL) ra_alloc(rtr->allocator, 0); for (ra = rtr->allocator; ra != NULL; ra = ra->next) { if (ra->sz - (ra->nextp - ra->memory_region) >= sz) { retp = ra->nextp; ra->nextp += sz; return retp; } if (ra->next == NULL) ra_alloc(ra->next, sz); } /* this should be unreachable code */ return NULL; } /** * strdup using ra_malloc, e.g. get memory from the allocator associated * to the given router. */ static char * ra_strdup(router *rtr, const char *s) { size_t sz = strlen(s) + 1; char *m = ra_malloc(rtr, sz); if (m == NULL) return m; memcpy(m, s, sz); return m; } /** * Frees routes and clusters. */ static void router_free_intern(cluster *clusters, route *routes) { while (routes != NULL) { if (routes->matchtype == REGEX) regfree(&routes->rule); if (routes->next == NULL || routes->next->dests != routes->dests) { while (routes->dests != NULL) { if (routes->dests->cl->type == GROUP || routes->dests->cl->type == AGGRSTUB || routes->dests->cl->type == STATSTUB) router_free_intern(NULL, routes->dests->cl->members.routes); if (routes->dests->cl->type == VALIDATION) router_free_intern(NULL, routes->dests->cl->members.validation->rule); routes->dests = routes->dests->next; } } routes = routes->next; } while (clusters != NULL) { switch (clusters->type) { case CARBON_CH: case FNV1A_CH: case JUMP_CH: ch_free(clusters->members.ch->ring); break; case FORWARD: case FILELOG: case FILELOGIP: case BLACKHOLE: case ANYOF: case FAILOVER: /* no special structures to free */ break; case GROUP: case AGGRSTUB: case STATSTUB: case VALIDATION: /* handled at the routes above */ break; case AGGREGATION: /* aggregators starve when they get no more input */ break; case REWRITE: /* everything is in the allocators */ break; } clusters = clusters->next; } } /** * Examines pattern and sets matchtype and rule or strmatch in route. */ static int determine_if_regex(route *r, char *pat, int flags) { /* try and see if we can avoid using a regex match, for * it is simply very slow/expensive to do so: most of * the time, people don't need fancy matching rules */ char patbuf[8192]; char *e = pat; char *pb = patbuf; char escape = 0; r->matchtype = CONTAINS; r->nmatch = 0; if (flags & REG_FORCE) { flags &= ~REG_NOSUB; r->matchtype = REGEX; } if (*e == '^' && r->matchtype == CONTAINS) { e++; r->matchtype = STARTS_WITH; } for (; *e != '\0'; e++) { switch (*e) { case '\\': if (escape) *pb++ = *e; escape = !escape; break; case '.': case '^': case '*': case '+': if (!escape) r->matchtype = REGEX; *pb++ = *e; escape = 0; break; case '$': if (!escape && e[1] == '\0') { if (r->matchtype == STARTS_WITH) { r->matchtype = MATCHES; } else { r->matchtype = ENDS_WITH; } } else { r->matchtype = REGEX; } escape = 0; break; default: if ( !escape && ( (*e == '_') || (*e == '-') || (*e >= '0' && *e <= '9') || (*e >= 'a' && *e <= 'z') || (*e >= 'A' && *e <= 'Z') ) ) { *pb++ = *e; } else { r->matchtype = REGEX; } escape = 0; break; } if (pb - patbuf == sizeof(patbuf)) r->matchtype = REGEX; if (r->matchtype == REGEX) break; } if (r->matchtype != REGEX) { *pb = '\0'; r->strmatch = strdup(patbuf); r->pattern = strdup(pat); } else { int ret = regcomp(&r->rule, pat, flags & ~REG_FORCE); if (ret != 0) return ret; /* allow use of regerror */ r->strmatch = NULL; r->pattern = strdup(pat); if (((flags & REG_NOSUB) == 0 && r->rule.re_nsub > 0) || flags & REG_FORCE) { /* we need +1 because position 0 contains the entire * expression */ r->nmatch = r->rule.re_nsub + 1; if (r->nmatch > RE_MAX_MATCHES) { logerr("determine_if_regex: too many match groups, " "please increase RE_MAX_MATCHES in router.h\n"); free(r->pattern); return REG_ESPACE; /* lie closest to the truth */ } } } return 0; } /** * Populates the routing tables by reading the config file. */ router * router_readconfig(router *orig, const char *path, size_t queuesize, size_t batchsize, int maxstalls, unsigned short iotimeout, unsigned int sockbufsize) { FILE *cnf; char *buf; size_t len = 0; char *p; struct stat st; struct addrinfo *saddrs; char matchcatchallfound = 0; router *ret = NULL; cluster *cl = NULL; aggregator *a = NULL; route *r = NULL; /* if there is no config, don't try anything */ if (stat(path, &st) == -1) { logerr("unable to stat file '%s': %s\n", path, strerror(errno)); return NULL; } if (orig == NULL) { /* get a return structure (and allocator) in place */ if ((ret = malloc(sizeof(router))) == NULL) { logerr("malloc failed for router return struct\n"); return NULL; } ret->allocator = NULL; ret->collector_stub = NULL; ret->routes = NULL; ret->aggregators = NULL; ret->srvrs = NULL; ret->clusters = NULL; /* create virtual blackhole cluster */ cl = ra_malloc(ret, sizeof(cluster)); if (cl == NULL) { logerr("malloc failed for blackhole cluster\n"); router_free(ret); return NULL; } cl->name = ra_strdup(ret, "blackhole"); cl->type = BLACKHOLE; cl->members.forward = NULL; cl->next = NULL; ret->clusters = cl; } else { ret = orig; /* position a, cl and r at the end of chains */ for (a = ret->aggregators; a != NULL && a->next != NULL; a = a->next) ; for (cl = ret->clusters; cl->next != NULL; cl = cl->next) ; for (r = ret->routes; r != NULL && r->next != NULL; r = r->next) ; } if ((buf = ra_malloc(ret, st.st_size + 1)) == NULL) { logerr("malloc failed for config file buffer\n"); return NULL; } if ((cnf = fopen(path, "r")) == NULL) { logerr("failed to open config file '%s': %s\n", path, strerror(errno)); return NULL; } while ((len = fread(buf + len, 1, st.st_size - len, cnf)) != 0) ; buf[st.st_size] = '\0'; fclose(cnf); /* remove all comments to ease parsing below */ p = buf; for (; *p != '\0'; p++) if (*p == '#') for (; *p != '\0' && *p != '\n'; p++) *p = ' '; p = buf; do { for (; *p != '\0' && isspace(*p); p++) ; if (*p == '\0') break; if (strncmp(p, "cluster", 7) == 0 && isspace(*(p + 7))) { /* group config */ servers *w; char *name; char useall = 0; p += 8; for (; *p != '\0' && isspace(*p); p++) ; name = p; for (; *p != '\0' && !isspace(*p); p++) ; if (*p == '\0') { logerr("unexpected end of file after 'cluster'\n"); router_free(ret); return NULL; } *p++ = '\0'; if ((cl = cl->next = ra_malloc(ret, sizeof(cluster))) == NULL) { logerr("malloc failed for cluster '%s'\n", name); router_free(ret); return NULL; } cl->name = ra_strdup(ret, name); cl->next = NULL; for (; *p != '\0' && isspace(*p); p++) ; if ( (strncmp(p, "carbon_ch", 9) == 0 && isspace(*(p + 9))) || (strncmp(p, "fnv1a_ch", 8) == 0 && isspace(*(p + 8))) || (strncmp(p, "jump_fnv1a_ch", 13) == 0 && isspace(*(p + 13))) ) { int replcnt = 1; enum clusttype chtype = *p == 'c' ? CARBON_CH : *p == 'f' ? FNV1A_CH : JUMP_CH; p += chtype == CARBON_CH ? 10 : chtype == FNV1A_CH ? 9 : 14; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "replication", 11) == 0 && isspace(*(p + 11))) { char *repl; p += 12; for (; *p != '\0' && isspace(*p); p++) ; repl = p; /* parse int */ for (; *p != '\0' && !isspace(*p); p++) ; if (*p == '\0') { logerr("unexpected end of file after " "'replication %s' for cluster %s\n", repl, name); router_free(ret); return NULL; } *p++ = '\0'; if ((replcnt = atoi(repl)) == 0) replcnt = 1; } cl->type = chtype; cl->members.ch = ra_malloc(ret, sizeof(chashring)); if (cl->members.ch == NULL) { logerr("malloc failed for ch in cluster '%s'\n", name); router_free(ret); return NULL; } cl->members.ch->repl_factor = (unsigned char)replcnt; cl->members.ch->ring = ch_new(chtype == CARBON_CH ? CARBON : chtype == FNV1A_CH ? FNV1a : JUMP_FNV1a); } else if (strncmp(p, "forward", 7) == 0 && isspace(*(p + 7))) { p += 8; cl->type = FORWARD; cl->members.forward = NULL; } else if (strncmp(p, "any_of", 6) == 0 && isspace(*(p + 6))) { p += 7; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "useall", 6) == 0 && isspace(*(p + 6))) { p += 7; useall = 1; } cl->type = ANYOF; cl->members.anyof = NULL; } else if (strncmp(p, "failover", 8) == 0 && isspace(*(p + 8))) { p += 9; cl->type = FAILOVER; cl->members.anyof = NULL; } else if (strncmp(p, "file", 4) == 0 && isspace(*(p + 4))) { p += 5; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "ip", 2) == 0 && isspace(*(p + 2))) { p += 3; for (; *p != '\0' && isspace(*p); p++) ; cl->type = FILELOGIP; } else { cl->type = FILELOG; } cl->members.forward = NULL; } else { char *type = p; for (; *p != '\0' && !isspace(*p); p++) ; *p = 0; logerr("unknown cluster type '%s' for cluster %s\n", type, name); router_free(ret); return NULL; } /* parse ips */ for (; *p != '\0' && isspace(*p); p++) ; w = NULL; do { char termchr; char *lastcolon = NULL; char *ip = p; char *inst = NULL; char *proto = "tcp"; int port = 2003; server *newserver = NULL; struct addrinfo hint; char sport[8]; int err; struct addrinfo *walk = NULL; struct addrinfo *next = NULL; char hnbuf[256]; for (; *p != '\0' && !isspace(*p) && *p != ';'; p++) { if (*p == ':' && inst == NULL) lastcolon = p; if (*p == '=' && inst == NULL) inst = p; } if (*p == '\0') { logerr("unexpected end of file at '%s' " "for cluster %s\n", ip, name); router_free(ret); return NULL; } termchr = *p; *p = '\0'; if (cl->type == CARBON_CH || cl->type == FNV1A_CH || cl->type == JUMP_CH) { if (inst != NULL) { *inst = '\0'; p = inst++; } if (inst == ip) inst = NULL; } if (*(p - 1) == ']') lastcolon = NULL; if (lastcolon != NULL) { char *endp = NULL; *lastcolon = '\0'; port = (int)strtol(lastcolon + 1, &endp, 10); if (port == 0 || endp != p) { logerr("expected port, or unexpected data at " "'%s' for cluster %s\n", lastcolon + 1, name); router_free(ret); return NULL; } } if (*ip == '[') { ip++; if (lastcolon != NULL && *(lastcolon - 1) == ']') { *(lastcolon - 1) = '\0'; } else if (lastcolon == NULL && *(p - 1) == ']') { *(p - 1) = '\0'; } else { logerr("expected ']' at '%s' " "for cluster %s\n", ip, name); router_free(ret); return NULL; } } if (inst != NULL) p = inst + strlen(inst); if (isspace(termchr)) { p++; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "proto", 5) == 0 && isspace(*(p + 5))) { p += 6; for (; *p != '\0' && isspace(*p); p++) ; proto = p; for (; *p != '\0' && !isspace(*p) && *p != ';'; p++) ; termchr = *p; *p = '\0'; if (strcmp(proto, "tcp") != 0 && strcmp(proto, "udp") != 0) { logerr("expected 'udp' or 'tcp' after " "'proto' at '%s' for cluster %s\n", proto, name); router_free(ret); return NULL; } } else { termchr = *p; } } if (cl->type != FILELOG && cl->type != FILELOGIP) { /* resolve host/IP */ memset(&hint, 0, sizeof(hint)); hint.ai_family = PF_UNSPEC; hint.ai_socktype = *proto == 'u' ? SOCK_DGRAM : SOCK_STREAM; hint.ai_protocol = *proto == 'u' ? IPPROTO_UDP : IPPROTO_TCP; hint.ai_flags = AI_NUMERICSERV; snprintf(sport, sizeof(sport), "%u", port); /* for default */ if ((err = getaddrinfo(ip, sport, &hint, &saddrs)) != 0) { logerr("failed to resolve server %s:%s (%s) " "for cluster %s: %s\n", ip, sport, proto, name, gai_strerror(err)); router_free(ret); return NULL; } if (!useall && saddrs->ai_next != NULL) { /* take first result only */ freeaddrinfo(saddrs->ai_next); saddrs->ai_next = NULL; } } else { /* TODO: try to create/append to file */ proto = "file"; saddrs = (void *)1; } walk = saddrs; while (walk != NULL) { servers *s; /* disconnect from the rest to avoid double * frees by freeaddrinfo() in server_destroy() */ if (walk != (void *)1) { next = walk->ai_next; walk->ai_next = NULL; } if (useall) { /* unfold whatever we resolved, for human * readability issues */ if (walk->ai_family == AF_INET) { if (inet_ntop(walk->ai_family, &((struct sockaddr_in *)walk->ai_addr)->sin_addr, hnbuf, sizeof(hnbuf)) != NULL) ip = hnbuf; } else if (walk->ai_family == AF_INET6) { if (inet_ntop(walk->ai_family, &((struct sockaddr_in6 *)walk->ai_addr)->sin6_addr, hnbuf + 1, sizeof(hnbuf) - 2) != NULL) { hnbuf[0] = '['; /* space is reserved above */ strcat(hnbuf, "]"); ip = hnbuf; } } } newserver = NULL; for (s = ret->srvrs; s != NULL; s = s->next) { if (strcmp(server_ip(s->server), ip) == 0 && server_port(s->server) == port && (*proto != 't' || server_ctype(s->server) == CON_TCP) && (*proto != 'u' || server_ctype(s->server) == CON_UDP)) { newserver = s->server; s->refcnt++; break; } } if (newserver == NULL) { newserver = server_new(ip, (unsigned short)port, *proto == 'f' ? CON_FILE : *proto == 'u' ? CON_UDP : CON_TCP, walk == (void *)1 ? NULL : walk, queuesize, batchsize, maxstalls, iotimeout, sockbufsize); if (newserver == NULL) { logerr("failed to add server %s:%d (%s) " "to cluster %s: %s\n", ip, port, proto, name, strerror(errno)); router_free(ret); freeaddrinfo(saddrs); return NULL; } if (ret->srvrs == NULL) { s = ret->srvrs = ra_malloc(ret, sizeof(servers)); } else { for (s = ret->srvrs; s->next != NULL; s = s->next) ; s = s->next = ra_malloc(ret, sizeof(servers)); } s->next = NULL; s->refcnt = 1; s->server = newserver; } if (cl->type == CARBON_CH || cl->type == FNV1A_CH || cl->type == JUMP_CH) { if (w == NULL) { cl->members.ch->servers = w = ra_malloc(ret, sizeof(servers)); } else { w = w->next = ra_malloc(ret, sizeof(servers)); } if (w == NULL) { logerr("malloc failed for %s_ch %s\n", cl->type == CARBON_CH ? "carbon" : cl->type == FNV1A_CH ? "fnv1a" : "jump_fnv1a", ip); router_free(ret); freeaddrinfo(saddrs); return NULL; } w->next = NULL; if (s->refcnt > 1) { char *sinst = server_instance(newserver); if (sinst == NULL && inst != NULL) { logerr("cannot set instance '%s' for " "server %s:%d: server was previously " "defined without instance\n", inst, server_ip(newserver), server_port(newserver)); router_free(ret); freeaddrinfo(saddrs); return NULL; } else if (sinst != NULL && inst == NULL) { logerr("cannot define server %s:%d without " "instance: server was previously " "defined with instance '%s'\n", server_ip(newserver), server_port(newserver), sinst); router_free(ret); freeaddrinfo(saddrs); return NULL; } else if (sinst != NULL && inst != NULL && strcmp(sinst, inst) != 0) { logerr("cannot set instance '%s' for " "server %s:%d: server was previously " "defined with instance '%s'\n", inst, server_ip(newserver), server_port(newserver), sinst); router_free(ret); freeaddrinfo(saddrs); return NULL; } /* else: sinst == inst == NULL */ } if (inst != NULL) server_set_instance(newserver, inst); w->server = newserver; cl->members.ch->ring = ch_addnode( cl->members.ch->ring, w->server); if (cl->members.ch->ring == NULL) { logerr("failed to add server %s:%d " "to ring for cluster %s: out of memory\n", ip, port, name); router_free(ret); freeaddrinfo(saddrs); return NULL; } } else if (cl->type == FORWARD || cl->type == ANYOF || cl->type == FAILOVER || cl->type == FILELOG || cl->type == FILELOGIP) { if (w == NULL) { w = ra_malloc(ret, sizeof(servers)); } else { w = w->next = ra_malloc(ret, sizeof(servers)); } if (w == NULL) { logerr("malloc failed for %s %s\n", cl->type == FORWARD ? "forward" : cl->type == ANYOF ? "any_of" : cl->type == FAILOVER ? "failover" : "file", ip); router_free(ret); freeaddrinfo(saddrs); return NULL; } w->next = NULL; w->server = newserver; if ((cl->type == FORWARD || cl->type == FILELOG || cl->type == FILELOGIP) && cl->members.forward == NULL) cl->members.forward = w; if (cl->type == ANYOF || cl->type == FAILOVER) { if (s->refcnt > 1) { logerr("cannot share server %s:%d with " "any_of/failover cluster '%s'\n", server_ip(newserver), server_port(newserver), cl->name); router_free(ret); freeaddrinfo(saddrs); return NULL; } if (cl->members.anyof == NULL) { cl->members.anyof = ra_malloc(ret, sizeof(serverlist)); if (cl->members.anyof == NULL) { logerr("malloc failed for %s anyof list\n", ip); router_free(ret); freeaddrinfo(saddrs); return NULL; } cl->members.anyof->count = 1; cl->members.anyof->servers = NULL; cl->members.anyof->list = w; } else { cl->members.anyof->count++; } } } walk = next; } *p = termchr; for (; *p != '\0' && isspace(*p); p++) ; } while (*p != ';'); p++; /* skip over ';' */ if (cl->type == ANYOF || cl->type == FAILOVER) { size_t i = 0; cl->members.anyof->servers = ra_malloc(ret, sizeof(server *) * cl->members.anyof->count); if (cl->members.anyof->servers == NULL) { logerr("malloc failed for anyof servers\n"); router_free(ret); return NULL; } for (w = cl->members.anyof->list; w != NULL; w = w->next) cl->members.anyof->servers[i++] = w->server; for (w = cl->members.anyof->list; w != NULL; w = w->next) { server_add_secondaries(w->server, cl->members.anyof->servers, cl->members.anyof->count); if (cl->type == FAILOVER) server_set_failover(w->server); } } else if (cl->type == CARBON_CH || cl->type == FNV1A_CH || cl->type == JUMP_CH) { /* check that replication count is actually <= the * number of servers */ size_t i = 0; for (w = cl->members.ch->servers; w != NULL; w = w->next) i++; if (i < cl->members.ch->repl_factor) { logerr("invalid cluster '%s': replication count (%zu) is " "larger than the number of servers (%zu)\n", name, cl->members.ch->repl_factor, i); router_free(ret); return NULL; } } } else if (strncmp(p, "match", 5) == 0 && isspace(*(p + 5))) { /* match rule */ char *pat; char *dest; char stop = -1; cluster *w; route *m = NULL; destinations *d = NULL; destinations *dw = NULL; p += 6; for (; *p != '\0' && isspace(*p); p++) ; do { pat = p; for (; *p != '\0' && !isspace(*p); p++) ; if (*p == '\0') { logerr("unexpected end of file after 'match'\n"); router_free(ret); return NULL; } *p++ = '\0'; for (; *p != '\0' && isspace(*p); p++) ; if (r == NULL) { ret->routes = r = ra_malloc(ret, sizeof(route)); } else { r = r->next = ra_malloc(ret, sizeof(route)); } if (r == NULL) { logerr("malloc failed for match '%s'\n", pat); router_free(ret); return NULL; } r->next = NULL; if (m == NULL) m = r; if (strcmp(pat, "*") == 0) { r->pattern = NULL; r->strmatch = NULL; r->matchtype = MATCHALL; } else { int err = determine_if_regex(r, pat, REG_EXTENDED | REG_NOSUB); if (err != 0) { char ebuf[512]; regerror(err, &r->rule, ebuf, sizeof(ebuf)); logerr("invalid expression '%s' for match: %s\n", pat, ebuf); router_free(ret); return NULL; } } } while ( (strncmp(p, "send", 4) != 0 || !isspace(*(p + 4))) && (strncmp(p, "validate", 8) != 0 || !isspace(*(p + 8)))); if (*p == 'v') { p += 9; if (dw == NULL) { dw = d = ra_malloc(ret, sizeof(destinations)); } else { d = d->next = ra_malloc(ret, sizeof(destinations)); } if (d == NULL) { logerr("out of memory allocating new validation " "for 'match %s'\n", pat); router_free(ret); return NULL; } d->next = NULL; if ((d->cl = ra_malloc(ret, sizeof(cluster))) == NULL) { logerr("malloc failed for validation rule in 'match %s'\n", pat); router_free(ret); return NULL; } d->cl->name = NULL; d->cl->type = VALIDATION; d->cl->next = NULL; d->cl->members.validation = ra_malloc(ret, sizeof(validate)); if (d->cl->members.validation == NULL) { logerr("malloc failed for validation member in " "'match %s'\n", pat); router_free(ret); return NULL; } for (; *p != '\0' && isspace(*p); p++) ; pat = p; do { for (; *p != '\0' && !isspace(*p); p++) ; } while (*p != '\0' && *(p - 1) == '\\' && *(p++) != '\0'); if (*p == '\0') { logerr("unexpected end of file after 'validate'\n"); router_free(ret); return NULL; } *p++ = '\0'; for (; *p != '\0' && isspace(*p); p++) ; if (strcmp(pat, "*") == 0) { logerr("cannot use '*' as validate expression\n"); router_free(ret); return NULL; } else { route *rule = ra_malloc(ret, sizeof(route)); int err; if (rule == NULL) { logerr("malloc failed for validate '%s'\n", pat); router_free(ret); return NULL; } rule->next = NULL; rule->dests = NULL; d->cl->members.validation->rule = rule; err = determine_if_regex(rule, pat, REG_EXTENDED | REG_NOSUB); if (err != 0) { char ebuf[512]; regerror(err, &rule->rule, ebuf, sizeof(ebuf)); logerr("invalid expression '%s' for validate: %s\n", pat, ebuf); router_free(ret); return NULL; } } if (strncmp(p, "else", 4) != 0 || !isspace(*(p + 4))) { logerr("expected 'else' after validate %s\n", pat); router_free(ret); return NULL; } p += 5; for (; *p != '\0' && isspace(*p); p++) ; if ((strncmp(p, "drop", 4) != 0 || !isspace(*(p + 4))) && (strncmp(p, "log", 3) != 0 || !isspace(*(p + 3)))) { logerr("expected 'drop' or 'log' after validate %s else\n", pat); router_free(ret); return NULL; } if (*p == 'd') { p += 5; d->cl->members.validation->action = VAL_DROP; } else { p += 4; d->cl->members.validation->action = VAL_LOG; } for (; *p != '\0' && isspace(*p); p++) ; if (*p == '\0') { logerr("unexpected end of file after validate %s\n", pat); router_free(ret); return NULL; } if (strncmp(p, "stop", 4) == 0) { if (isspace(*(p + 4))) { p += 5; for (; *p != '\0' && isspace(*p); p++) ; } else if (*(p + 4) != ';') { logerr("expected ';' after 'stop' for validate %s\n", pat); router_free(ret); return NULL; } } if (*p == ';') { p++; do { m->dests = dw; /* always ignore stop here, else the whole * contruct makes no sense */ m->stop = 0; } while (m != r && (m = m->next) != NULL); continue; } if (strncmp(p, "send", 4) != 0 || !isspace(*(p + 4))) { logerr("expected 'send to' after validate %s\n", pat); router_free(ret); return NULL; } } p += 5; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "to", 2) != 0 || !isspace(*(p + 2))) { logerr("expected 'to' after 'send' for match %s\n", r->pattern ? r->pattern : "*"); router_free(ret); return NULL; } p += 3; for (; *p != '\0' && isspace(*p); p++) ; do { char save; dest = p; for (; *p != '\0' && !isspace(*p) && *p != ';'; p++) ; if (*p == '\0') break; save = *p; *p = '\0'; /* lookup dest */ for (w = ret->clusters; w != NULL; w = w->next) { if (w->type != GROUP && w->type != AGGRSTUB && w->type != STATSTUB && w->type != AGGREGATION && w->type != REWRITE && w->type != VALIDATION && strcmp(w->name, dest) == 0) break; } if (w == NULL) { logerr("no such cluster '%s' for 'match %s'\n", dest, r->pattern ? r->pattern : "*"); router_free(ret); return NULL; } if (dw == NULL) { dw = d = ra_malloc(ret, sizeof(destinations)); } else { d = d->next = ra_malloc(ret, sizeof(destinations)); } if (d == NULL) { logerr("out of memory allocating new destination '%s' " "for 'match %s'\n", dest, r->pattern ? r->pattern : "*"); router_free(ret); return NULL; } d->cl = w; d->next = NULL; *p = save; for (; *p != '\0' && isspace(*p); p++) ; } while (*p != ';' && !(strncmp(p, "stop", 4) == 0 && (isspace(*(p + 4)) || *(p + 4) == ';'))); if (*p == '\0') { logerr("unexpected end of file after 'send to %s'\n", dest); router_free(ret); return NULL; } else if (*p == ';') { stop = 0; } else { /* due to strncmp above, this must be stop */ p += 4; stop = 1; } /* find the closing ';' */ for (; *p != '\0' && isspace(*p); p++) ; if (*p != ';') { logerr("expected ';' after match %s\n", pat); router_free(ret); return NULL; } p++; /* fill in the destinations for all the routes */ do { m->dests = dw; m->stop = w->type == BLACKHOLE ? 1 : stop; } while (m != r && (m = m->next) != NULL); if (matchcatchallfound) { logerr("warning: match %s will never be matched " "due to preceding match * ... stop\n", r->pattern == NULL ? "*" : r->pattern); } if (r->matchtype == MATCHALL && r->stop) matchcatchallfound = 1; } else if (strncmp(p, "aggregate", 9) == 0 && isspace(*(p + 9))) { /* aggregation rule */ char *type; char *pat; char *num; enum _aggr_timestamp tswhen = TS_END; cluster *w; int err; int intv; int exp; char stop = 0; route *m = NULL; destinations *d = NULL; destinations *dw = NULL; p += 10; for (; *p != '\0' && isspace(*p); p++) ; w = ra_malloc(ret, sizeof(cluster)); if (w == NULL) { logerr("malloc failed for aggregate\n"); router_free(ret); return NULL; } w->name = NULL; w->type = AGGREGATION; w->next = NULL; do { pat = p; for (; *p != '\0' && !isspace(*p); p++) ; if (*p == '\0') { logerr("unexpected end of file after 'aggregate'\n"); router_free(ret); return NULL; } *p++ = '\0'; for (; *p != '\0' && isspace(*p); p++) ; if (r == NULL) { ret->routes = r = ra_malloc(ret, sizeof(route)); } else { r = r->next = ra_malloc(ret, sizeof(route)); } if (r == NULL) { logerr("malloc failed for aggregate route\n"); router_free(ret); return NULL; } r->next = NULL; if (m == NULL) m = r; err = determine_if_regex(r, pat, REG_EXTENDED); if (err != 0) { char ebuf[512]; regerror(err, &r->rule, ebuf, sizeof(ebuf)); logerr("invalid expression '%s' " "for aggregation: %s\n", pat, ebuf); router_free(ret); return NULL; } } while (strncmp(p, "every", 5) != 0 || !isspace(*(p + 5))); p += 6; for (; *p != '\0' && isspace(*p); p++) ; num = p; for (; *p != '\0' && isdigit(*p); p++) ; if (*p == '\0') { logerr("unexpected end of file after 'every %s'\n", num); router_free(ret); return NULL; } if (!isspace(*p)) { logerr("unexpected character '%c', " "expected number after 'every'\n", *p); router_free(ret); return NULL; } *p++ = '\0'; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "seconds", 7) != 0 || !isspace(*(p + 7))) { logerr("expected 'seconds' after 'every %s'\n", num); router_free(ret); return NULL; } p += 8; intv = atoi(num); if (intv == 0) { logerr("interval must be non-zero\n"); router_free(ret); return NULL; } for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "expire", 6) != 0 || !isspace(*(p + 6))) { logerr("expected 'expire after' after 'every %s seconds\n", num); router_free(ret); return NULL; } p += 7; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "after", 5) != 0 || !isspace(*(p + 5))) { logerr("expected 'after' after 'expire'\n"); router_free(ret); return NULL; } p += 6; for (; *p != '\0' && isspace(*p); p++) ; num = p; for (; *p != '\0' && isdigit(*p); p++) ; if (*p == '\0') { logerr("unexpected end of file after 'expire after %s'\n", num); router_free(ret); return NULL; } if (!isspace(*p)) { logerr("unexpected character '%c', " "expected number after 'expire after'\n", *p); router_free(ret); return NULL; } *p++ = '\0'; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "seconds", 7) != 0 || !isspace(*(p + 7))) { logerr("expected 'seconds' after 'expire after %s'\n", num); router_free(ret); return NULL; } p += 8; exp = atoi(num); if (exp == 0) { logerr("expire must be non-zero\n"); router_free(ret); return NULL; } if (exp < intv) { logerr("expire (%d) must be greater than interval (%d)\n", exp, intv); router_free(ret); return NULL; } for (; *p != '\0' && isspace(*p); p++) ; /* optional timestamp bit */ if (strncmp(p, "timestamp", 9) == 0 && isspace(*(p + 9))) { p += 10; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "at", 2) != 0 || !isspace(*(p + 2))) { logerr("expected 'at' after 'timestamp'\n"); router_free(ret); return NULL; } p += 3; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "start", 5) == 0 && isspace(*(p + 5))) { p += 6; tswhen = TS_START; } else if (strncmp(p, "middle", 6) == 0 && isspace(*(p + 6))) { p += 7; tswhen = TS_MIDDLE; } else if (strncmp(p, "end", 3) == 0 && isspace(*(p + 3))) { p += 4; tswhen = TS_END; } else { logerr("expected 'start', 'middle' or 'end' " "after 'timestamp at'\n"); router_free(ret); return NULL; } for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "of", 2) != 0 || !isspace(*(p + 2))) { logerr("expected 'of' after 'timestamp at ...'\n"); router_free(ret); return NULL; } p += 3; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "bucket", 6) != 0 || !isspace(*(p + 6))) { logerr("expected 'bucket' after 'timestamp at ... of'\n"); router_free(ret); return NULL; } p += 7; for (; *p != '\0' && isspace(*p); p++) ; } w->members.aggregation = aggregator_new(intv, exp, tswhen); if (w->members.aggregation == NULL) { logerr("out of memory while allocating new aggregator\n"); router_free(ret); return NULL; } if (a == NULL) { ret->aggregators = a = w->members.aggregation; } else { a = a->next = w->members.aggregation; } do { if (strncmp(p, "compute", 7) != 0 || !isspace(*(p + 7))) { logerr("expected 'compute' at: %.20s\n", p); router_free(ret); return NULL; } p += 8; pat = p; for (; *p != '\0' && !isspace(*p); p++) ; if (*p == '\0') { logerr("unexpected end of file after 'compute'\n"); router_free(ret); return NULL; } *p++ = '\0'; type = pat; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "write", 5) != 0 || !isspace(*(p + 5))) { logerr("expected 'write to' after 'compute %s'\n", pat); router_free(ret); return NULL; } p += 6; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "to", 2) != 0 || !isspace(*(p + 2))) { logerr("expected 'write to' after 'compute %s'\n", pat); router_free(ret); return NULL; } p += 3; for (; *p != '\0' && isspace(*p); p++) ; pat = p; for (; *p != '\0' && !isspace(*p); p++) ; if (*p == '\0') { logerr("unexpected end of file after 'write to'\n"); router_free(ret); return NULL; } *p++ = '\0'; if (aggregator_add_compute(w->members.aggregation, pat, type) != 0) { logerr("expected sum, count, max, min, average, " "median, percentile<%>, variance or stddev " "after 'compute', got '%s'\n", type); router_free(ret); return NULL; } for (; *p != '\0' && isspace(*p); p++) ; } while (*p != ';' && !(strncmp(p, "send", 4) == 0 && isspace(*(p + 4))) && !(strncmp(p, "stop", 4) == 0 && (isspace(*(p + 4)) || *(p + 4) == ';'))); if (strncmp(p, "send", 4) == 0 && isspace(*(p + 4))) { cluster *cw; p += 5; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "to", 2) != 0 || !isspace(*(p + 2))) { logerr("expected 'to' after 'send' for aggregate\n"); router_free(ret); return NULL; } p += 3; for (; *p != '\0' && isspace(*p); p++) ; do { char save; char *dest = p; for (; *p != '\0' && !isspace(*p) && *p != ';'; p++) ; if (*p == '\0') break; save = *p; *p = '\0'; /* lookup dest */ for (cw = ret->clusters; cw != NULL; cw = cw->next) { if (cw->type != GROUP && cw->type != AGGRSTUB && cw->type != STATSTUB && cw->type != AGGREGATION && cw->type != REWRITE && cw->type != VALIDATION && strcmp(cw->name, dest) == 0) break; } if (cw == NULL) { logerr("no such cluster '%s' for aggregate\n", dest); router_free(ret); return NULL; } if (dw == NULL) { dw = d = ra_malloc(ret, sizeof(destinations)); } else { d = d->next = ra_malloc(ret, sizeof(destinations)); } if (d == NULL) { logerr("out of memory allocating new destination '%s' " "for aggregation\n", dest); router_free(ret); return NULL; } d->cl = cw; d->next = NULL; *p = save; for (; *p != '\0' && isspace(*p); p++) ; } while (*p != ';' && !(strncmp(p, "stop", 4) == 0 && (isspace(*(p + 4)) || *(p + 4) == ';'))); } if (strncmp(p, "stop", 4) == 0 && (isspace(*(p + 4)) || *(p + 4) == ';')) { p += 4; stop = 1; for (; *p != '\0' && isspace(*p); p++) ; } if (*p == '\0') { logerr("unexpected end of file after 'aggregate %s'\n", r->pattern); router_free(ret); return NULL; } if (*p != ';') { logerr("expected ';' after aggregate %s\n", r->pattern); router_free(ret); return NULL; } p++; /* add cluster to the list of clusters */ cl = cl->next = w; /* fill in the destinations for all the routes, this is for * printing and free-ing */ do { m->dests = ra_malloc(ret, sizeof(destinations)); if (m->dests == NULL) { logerr("malloc failed for aggregation destinations\n"); router_free(ret); return NULL; } m->dests->cl = w; m->dests->next = dw; m->stop = stop; } while (m != r && (m = m->next) != NULL); if (dw != NULL) { char stubname[48]; m = ra_malloc(ret, sizeof(route)); if (m == NULL) { logerr("malloc failed for aggregation destination stub\n"); router_free(ret); return NULL; } m->pattern = NULL; m->strmatch = NULL; m->dests = dw; m->stop = 1; m->matchtype = MATCHALL; m->next = NULL; /* inject stub route for dests */ d = ra_malloc(ret, sizeof(destinations)); if (d == NULL) { logerr("malloc failed for aggr stub route\n"); router_free(ret); return NULL; } d->next = NULL; cl = cl->next = d->cl = ra_malloc(ret, sizeof(cluster)); if (cl == NULL) { logerr("malloc failed for aggr stub destinations\n"); router_free(ret); return NULL; } cl->name = NULL; cl->type = AGGRSTUB; cl->members.routes = m; cl->next = NULL; snprintf(stubname, sizeof(stubname), STUB_AGGR "%p__", w->members.aggregation); m = ra_malloc(ret, sizeof(route)); if (m == NULL) { logerr("malloc failed for catch stub aggr route\n"); router_free(ret); return NULL; } m->pattern = ra_strdup(ret, stubname); m->strmatch = ra_strdup(ret, stubname); if (m->pattern == NULL || m->strmatch == NULL) { logerr("malloc failed for catch stub aggr route pattern\n"); router_free(ret); return NULL; } m->dests = d; m->stop = 1; m->matchtype = STARTS_WITH; /* enforce first match to avoid interference */ m->next = ret->routes; ret->routes = m; aggregator_set_stub(w->members.aggregation, stubname); } if (matchcatchallfound) { logerr("warning: aggregate %s will never be matched " "due to preceding match * ... stop\n", r->pattern); } } else if (strncmp(p, "rewrite", 7) == 0 && isspace(*(p + 7))) { /* rewrite rule */ char *pat; char *replacement; int err; p += 8; for (; *p != '\0' && isspace(*p); p++) ; pat = p; for (; *p != '\0' && !isspace(*p); p++) ; if (*p == '\0') { logerr("unexpected end of file after 'rewrite'\n"); router_free(ret); return NULL; } *p++ = '\0'; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "into", 4) != 0 || !isspace(*(p + 4))) { logerr("expected 'into' after rewrite %s\n", pat); router_free(ret); return NULL; } p += 5; for (; *p != '\0' && isspace(*p); p++) ; replacement = p; for (; *p != '\0' && !isspace(*p) && *p != ';'; p++) ; if (*p == '\0') { logerr("unexpected end of file after 'into %s'\n", replacement); router_free(ret); return NULL; } else if (*p == ';') { *p++ = '\0'; } else { *p++ = '\0'; for (; *p != '\0' && isspace(*p) && *p != ';'; p++) ; if (*p != ';') { logerr("expected ';' after %s\n", replacement); router_free(ret); return NULL; } p++; } if (r == NULL) { ret->routes = r = ra_malloc(ret, sizeof(route)); } else { r = r->next = ra_malloc(ret, sizeof(route)); } if (r == NULL) { logerr("malloc failed for rewrite route\n"); router_free(ret); return NULL; } r->next = NULL; err = determine_if_regex(r, pat, REG_EXTENDED | REG_FORCE); if (err != 0) { char ebuf[512]; regerror(err, &r->rule, ebuf, sizeof(ebuf)); logerr("invalid expression '%s' for rewrite: %s\n", pat, ebuf); router_free(ret); return NULL; } if ((cl = cl->next = ra_malloc(ret, sizeof(cluster))) == NULL) { logerr("malloc failed for rewrite destination\n"); router_free(ret); return NULL; } cl->type = REWRITE; cl->name = NULL; cl->members.replacement = ra_strdup(ret, replacement); cl->next = NULL; if (cl->members.replacement == NULL) { logerr("malloc failed for rewrite cluster replacement\n"); router_free(ret); return NULL; } r->dests = ra_malloc(ret, sizeof(destinations)); if (r->dests == NULL) { logerr("malloc failed for rewrite destinations\n"); router_free(ret); return NULL; } r->dests->cl = cl; r->dests->next = NULL; r->stop = 0; r->next = NULL; if (matchcatchallfound) { logerr("warning: rewrite %s will never be matched " "due to preceding match * ... stop\n", r->pattern == NULL ? "*" : r->pattern); } } else if (strncmp(p, "send", 4) == 0 && isspace(*(p + 4))) { cluster *cw; destinations *d = NULL; destinations *dw = NULL; route *m; char stubname[48]; p += 4; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "statistics", 10) != 0 || !isspace(*(p + 10))) { logerr("expected 'statistics' after 'send'\n"); router_free(ret); return NULL; } p += 10; for (; *p != '\0' && isspace(*p); p++) ; if (strncmp(p, "to", 2) != 0 || !isspace(*(p + 2))) { logerr("expected 'to' after 'send statistics'\n"); router_free(ret); return NULL; } p += 2; if (ret->collector_stub != NULL) { logerr("duplicate 'send statistics to' not allowed, " "use multiple destinations instead\n"); router_free(ret); return NULL; } for (; *p != '\0' && isspace(*p); p++) ; do { char save; char *dest = p; for (; *p != '\0' && !isspace(*p) && *p != ';'; p++) ; if (*p == '\0' || *p == ';') break; save = *p; *p = '\0'; /* lookup dest */ for (cw = ret->clusters; cw != NULL; cw = cw->next) { if (cw->type != GROUP && cw->type != AGGRSTUB && cw->type != STATSTUB && cw->type != AGGREGATION && cw->type != REWRITE && cw->type != VALIDATION && strcmp(cw->name, dest) == 0) break; } if (cw == NULL) { logerr("no such cluster '%s' for send statistics to\n", dest); router_free(ret); return NULL; } if (dw == NULL) { dw = d = ra_malloc(ret, sizeof(destinations)); } else { d = d->next = ra_malloc(ret, sizeof(destinations)); } if (d == NULL) { logerr("out of memory allocating new destination '%s' " "for 'send statistics to'\n", dest); router_free(ret); return NULL; } d->cl = cw; d->next = NULL; *p = save; for (; *p != '\0' && isspace(*p); p++) ; } while (*p != ';' && !(strncmp(p, "stop", 4) == 0 && (isspace(*(p + 4)) || *(p + 4) == ';'))); if (strncmp(p, "stop", 4) == 0 && (isspace(*(p + 4)) || *(p + 4) == ';')) { p += 4; for (; *p != '\0' && isspace(*p); p++) ; } if (*p == ';') p++; if (dw == NULL) { logerr("expected cluster name after 'send statistics to'\n"); router_free(ret); return NULL; } /* create a top-rule with match for unique prefix and send * that to the defined targets */ m = ra_malloc(ret, sizeof(route)); if (m == NULL) { logerr("malloc failed for send statistics route\n"); router_free(ret); return NULL; } m->pattern = NULL; m->strmatch = NULL; m->dests = dw; m->stop = 1; m->matchtype = MATCHALL; m->next = NULL; /* inject stub route for dests */ d = ra_malloc(ret, sizeof(destinations)); if (d == NULL) { logerr("malloc failed for send statistics stub\n"); router_free(ret); return NULL; } d->next = NULL; cl = cl->next = d->cl = ra_malloc(ret, sizeof(cluster)); if (cl == NULL) { logerr("malloc failed for send statistics cluster\n"); router_free(ret); return NULL; } cl->name = NULL; cl->type = STATSTUB; cl->members.routes = m; cl->next = NULL; snprintf(stubname, sizeof(stubname), STUB_STATS "%p__", ret); m = ra_malloc(ret, sizeof(route)); if (m == NULL) { logerr("malloc failed for send statistics route stub\n"); router_free(ret); return NULL; } m->pattern = ra_strdup(ret, stubname); m->strmatch = ra_strdup(ret, stubname); if (m->pattern == NULL || m->strmatch == NULL) { logerr("malloc failed for send statistics stub pattern\n"); router_free(ret); return NULL; } m->dests = d; m->stop = 1; m->matchtype = STARTS_WITH; /* enforce first match to avoid interference */ m->next = ret->routes; ret->routes = m; if (r == NULL) r = ret->routes; ret->collector_stub = m->pattern; } else if (strncmp(p, "include", 7) == 0 && isspace(*(p + 7))) { char *name = NULL; char endchar; p += 7; for (; *p != '\0' && isspace(*p); p++) ; if (*p == '\0') { logerr("unexpected end of input after 'include'\n"); router_free(ret); return NULL; } name = p; for (; *p != '\0' && !isspace(*p) && *p != ';'; p++) ; if (*p == '\0') { logerr("unexpected end of input after 'include %s' " "(did you forget ';'?)\n", name); router_free(ret); return NULL; } endchar = *p; *p = '\0'; if (strchr(name, '*') || strchr(name, '?') || (strchr(name, '[') && strchr(name, ']'))) { /* include path is a glob pattern */ glob_t globbuf; char *globpath; size_t i; if ((i = glob(name, 0, NULL, &globbuf)) != 0) { switch (i) { case GLOB_NOSPACE: /* since we don't set a limit, we won't * reach it either */ logerr("out of memory while globbing files for " "pattern '%s'\n", name); router_free(ret); ret = NULL; break; case GLOB_ABORTED: /* we don't use a call-back, so this is * a real error of some sort */ logerr("unable to match any files from " "pattern '%s': %s\n", name, strerror(errno)); router_free(ret); ret = NULL; break; case GLOB_NOMATCH: /* we don't want to abort on failing globs */ logout("warning: pattern '%s' did not match " "any files\n", name); break; } } /* read all files matched by glob */ for (i = 0; i < globbuf.gl_pathc; i++) { globpath = globbuf.gl_pathv[i]; ret = router_readconfig(ret, globpath, queuesize, batchsize, maxstalls, iotimeout, sockbufsize); if (ret == NULL) break; } /* also after errors the globbuf structure is * initialised and might need free-ing */ globfree(&globbuf); } else { /* include path is a regular file path */ ret = router_readconfig(ret, name, queuesize, batchsize, maxstalls, iotimeout, sockbufsize); } if (ret == NULL) /* router_readconfig already barked and freed ret */ return NULL; /* the included file could have added new aggregates, * matches, or clusters, so adjust these chains to point to * the new end. */ for (a = ret->aggregators; a != NULL && a->next != NULL; a = a->next) ; for (cl = ret->clusters; cl->next != NULL; cl = cl->next) ; for (r = ret->routes; r != NULL && r->next != NULL; r = r->next) ; *p = endchar; for (; *p != '\0' && isspace(*p); p++) ; if (*p != ';') { logerr("expected ';' after 'include %s'\n", name); router_free(ret); return NULL; } p++; } else { /* garbage? */ logerr("unexpected input in config: %s\n", p); router_free(ret); return NULL; } } while (*p != '\0'); return ret; } typedef struct _block { char *pattern; size_t hash; size_t refcnt; size_t seqnr; route *firstroute; route *lastroute; struct _block *prev; struct _block *next; } block; /** * Tries to optimise the match and aggregation rules in such a way that * the number of matches for non-matching metrics are reduced. The * problem is that with many metrics flowing in, the time to perform * lots of regex matches is high. This is not too bad if that time * spent actually results in a metric being counted (aggregation) or * sent further (match), but it is when the metric would be discarded * for it did not match anything. * Hence, we employ a simple strategy to try and reduce the incoming * stream of metrics as soon as possible before performing the more * specific and expensive matches to confirm fit. */ void router_optimise(router *r) { char *p; char pblock[64]; char *b; route *rwalk; route *rnext; route *rlast; block *blocks; block *bwalk; block *bstart; block *blast; size_t bsum; size_t seq; /* avoid optimising anything if it won't pay off */ seq = 0; for (rwalk = r->routes; rwalk != NULL && seq < 50; rwalk = rwalk->next) seq++; if (seq < 50) return; /* Heuristic: the last part of the matching regex is the most * discriminating part of the metric. The last part is defined as a * block of characters matching [a-zA-Z_]+ at the end disregarding * any characters not matched by the previous expression. Then from * these last parts we create groups, that -- if having enough * members -- is used to reduce the amount of comparisons done * before determining that an input metric cannot match any * expression we have defined. */ seq = 0; blast = bstart = blocks = malloc(sizeof(block)); blocks->refcnt = 0; blocks->pattern = NULL; blocks->hash = 0; blocks->seqnr = seq++; blocks->prev = NULL; blocks->next = NULL; rlast = NULL; for (rwalk = r->routes; rwalk != NULL; rwalk = rnext) { /* matchall and rewrite rules cannot be in a group (issue #218) */ if (rwalk->matchtype == MATCHALL || rwalk->dests->cl->type == REWRITE) { blast->next = malloc(sizeof(block)); blast->next->prev = blast; blast = blast->next; blast->pattern = NULL; blast->hash = 0; blast->refcnt = 1; blast->seqnr = seq++; blast->firstroute = rwalk; blast->lastroute = rwalk; blast->next = NULL; rnext = rwalk->next; rwalk->next = NULL; bstart = blast; rlast = NULL; continue; } p = rwalk->pattern + strlen(rwalk->pattern); /* strip off chars that won't belong to a block */ bsum = 0; while ( p > rwalk->pattern && ( bsum > 0 || ( (*p < 'a' || *p > 'z') && (*p < 'A' || *p > 'Z') && *p != '_' ) ) ) { if (*p == ')' || *p == '(') { char esc = 0; char oe = *p; /* trim escapes, need to figure out if this is escaped * or not, in order to find the a matching open or not */ for (--p; p > rwalk->pattern && *p == '\\'; p--) esc = !esc; if (!esc) /* not escaped */ bsum += oe == ')' ? 1 : -1; continue; /* skip p-- */ } p--; } if (p == rwalk->pattern) { /* nothing we can do with a pattern like this */ blast->next = malloc(sizeof(block)); blast->next->prev = blast; blast = blast->next; blast->pattern = NULL; blast->hash = 0; blast->refcnt = 1; blast->seqnr = seq++; blast->firstroute = rwalk; blast->lastroute = rwalk; blast->next = NULL; rnext = rwalk->next; rwalk->next = NULL; bstart = blast; rlast = NULL; continue; } /* find the block */ bsum = 0; b = pblock; while ( p >= rwalk->pattern && b - pblock < sizeof(pblock) && ( (*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || *p == '_' ) ) { bsum += *p; *b++ = *p--; } *b = '\0'; b = pblock; if (strlen(b) < 3) { /* this probably isn't selective enough, don't put in a group */ blast->next = malloc(sizeof(block)); blast->next->prev = blast; blast = blast->next; blast->pattern = NULL; blast->hash = 0; blast->refcnt = 1; blast->seqnr = seq++; blast->firstroute = rwalk; blast->lastroute = rwalk; blast->next = NULL; rnext = rwalk->next; rwalk->next = NULL; bstart = blast; rlast = NULL; continue; } /* at this point, b points to the tail block in reverse, see if * we already had such tail in place */ for (bwalk = bstart; bwalk != NULL; bwalk = bwalk->next) { if (bwalk->hash != bsum || strcmp(bwalk->pattern, b) != 0) continue; break; } if (bwalk == NULL) { blast->next = malloc(sizeof(block)); blast->next->prev = blast; blast = blast->next; blast->pattern = strdup(b); blast->hash = bsum; blast->refcnt = 1; blast->seqnr = seq; blast->firstroute = rwalk; blast->lastroute = rwalk; blast->next = NULL; rnext = rwalk->next; rwalk->next = NULL; if (rwalk->stop) { bstart = blast; rlast = NULL; seq++; } continue; } bwalk->refcnt++; bwalk->lastroute = bwalk->lastroute->next = rwalk; rnext = rwalk->next; rwalk->next = NULL; if (rlast == NULL || strcmp(rlast->pattern, b) == 0) { if (rwalk->stop) { /* move this one to the end */ if (bwalk->next != NULL) { bwalk->prev->next = bwalk->next; blast = blast->next = bwalk; bwalk->next = NULL; } rlast = rwalk; } } else { bstart = blast; rlast = NULL; seq++; } } /* make loop below easier by appending a dummy (reuse the one from * start) */ blast = blast->next = blocks; blocks = blocks->next; blast->next = NULL; blast->seqnr = seq; rwalk = r->routes = NULL; seq = 1; /* create groups, if feasible */ for (bwalk = blocks; bwalk != NULL; bwalk = blast) { if (bwalk->seqnr != seq) { seq++; blast = bwalk; continue; } if (bwalk->refcnt == 0) { blast = bwalk->next; free(bwalk); continue; } else if (bwalk->refcnt < 3) { if (r->routes == NULL) { r->routes = bwalk->firstroute; } else { rwalk->next = bwalk->firstroute; } rwalk = bwalk->lastroute; blast = bwalk->next; free(bwalk->pattern); free(bwalk); } else { if (r->routes == NULL) { rwalk = r->routes = malloc(sizeof(route)); } else { rwalk = rwalk->next = malloc(sizeof(route)); } rwalk->pattern = NULL; rwalk->stop = 0; rwalk->matchtype = CONTAINS; rwalk->dests = malloc(sizeof(destinations)); rwalk->dests->cl = malloc(sizeof(cluster)); rwalk->dests->cl->name = bwalk->pattern; rwalk->dests->cl->type = GROUP; rwalk->dests->cl->members.routes = bwalk->firstroute; rwalk->dests->cl->next = NULL; rwalk->dests->next = NULL; rwalk->next = NULL; blast = bwalk->next; free(bwalk); } } } /** * Returns all (unique) servers from the cluster-configuration. */ server ** router_getservers(router *r) { server **ret; servers *s; int i; i = 0; for (s = r->srvrs; s != NULL; s = s->next) i++; ret = malloc(sizeof(server *) * (i + 1)); if (ret == NULL) return NULL; i = 0; for (s = r->srvrs; s != NULL; s = s->next) ret[i++] = s->server; /* ensure NULL-termination */ ret[i] = NULL; return ret; } /** * Returns all aggregators from this router configuration */ inline aggregator * router_getaggregators(router *rtr) { return rtr->aggregators; } /** * Returns the stub prefix to use for collector stats, or NULL if no * such thing should be employed. */ inline char * router_getcollectorstub(router *rtr) { return rtr->collector_stub; } /** * Mere debugging function to check if the configuration is picked up * alright. If all is set to false, aggregation rules won't be printed. * This comes in handy because aggregations usually come in the order of * thousands. */ void router_printconfig(router *rtr, FILE *f, char pmode) { cluster *c; route *r; servers *s; #define PPROTO \ server_ctype(s->server) == CON_UDP ? " proto udp" : "" for (c = rtr->clusters; c != NULL; c = c->next) { if (c->type == BLACKHOLE || c->type == REWRITE || c->type == GROUP || c->type == AGGREGATION || c->type == AGGRSTUB || c->type == STATSTUB) continue; fprintf(f, "cluster %s\n", c->name); if (c->type == FORWARD) { fprintf(f, " forward\n"); for (s = c->members.forward; s != NULL; s = s->next) fprintf(f, " %s:%d%s\n", server_ip(s->server), server_port(s->server), PPROTO); } else if (c->type == FILELOG || c->type == FILELOGIP) { fprintf(f, " file%s\n", c->type == FILELOGIP ? " ip" : ""); for (s = c->members.forward; s != NULL; s = s->next) fprintf(f, " %s\n", server_ip(s->server)); } else if (c->type == ANYOF || c->type == FAILOVER) { fprintf(f, " %s\n", c->type == ANYOF ? "any_of" : "failover"); for (s = c->members.anyof->list; s != NULL; s = s->next) fprintf(f, " %s:%d%s\n", server_ip(s->server), server_port(s->server), PPROTO); } else if (c->type == CARBON_CH || c->type == FNV1A_CH || c->type == JUMP_CH) { fprintf(f, " %s_ch replication %d\n", c->type == CARBON_CH ? "carbon" : c->type == FNV1A_CH ? "fnv1a" : "jump_fnv1a", c->members.ch->repl_factor); for (s = c->members.ch->servers; s != NULL; s = s->next) fprintf(f, " %s:%d%s%s%s\n", server_ip(s->server), server_port(s->server), server_instance(s->server) ? "=" : "", server_instance(s->server) ? server_instance(s->server) : "", PPROTO); } fprintf(f, " ;\n"); if (pmode & PMODE_HASH) { if (c->type == CARBON_CH || c->type == FNV1A_CH || c->type == JUMP_CH) { fprintf(f, "# hash ring for %s follows\n", c->name); ch_printhashring(c->members.ch->ring, f); } } } fprintf(f, "\n"); for (r = rtr->routes; r != NULL; r = r->next) { if (r->dests->cl->type == AGGREGATION) { cluster *aggr = r->dests->cl; struct _aggr_computes *ac; char stubname[48]; char percentile[16]; if (!(pmode & PMODE_AGGR)) continue; if (pmode & PMODE_STUB || r->dests->next == NULL) { stubname[0] = '\0'; } else { snprintf(stubname, sizeof(stubname), STUB_AGGR "%p__", aggr->members.aggregation); } fprintf(f, "aggregate"); if (r->next == NULL || r->next->dests->cl != aggr) { fprintf(f, " %s\n", r->pattern); } else { fprintf(f, "\n"); do { fprintf(f, " %s\n", r->pattern); } while (r->next != NULL && r->next->dests->cl == aggr && (r = r->next) != NULL); } fprintf(f, " every %u seconds\n" " expire after %u seconds\n" " timestamp at %s of bucket\n", aggr->members.aggregation->interval, aggr->members.aggregation->expire, aggr->members.aggregation->tswhen == TS_START ? "start" : aggr->members.aggregation->tswhen == TS_MIDDLE ? "middle" : aggr->members.aggregation->tswhen == TS_END ? "end" : ""); for (ac = aggr->members.aggregation->computes; ac != NULL; ac = ac->next) { snprintf(percentile, sizeof(percentile), "percentile%d", ac->percentile); fprintf(f, " compute %s write to\n" " %s\n", ac->type == SUM ? "sum" : ac->type == CNT ? "count" : ac->type == MAX ? "max" : ac->type == MIN ? "min" : ac->type == AVG ? "average" : ac->type == MEDN ? "median" : ac->type == PCTL ? percentile : ac->type == VAR ? "variance" : ac->type == SDEV ? "stddev" : "", ac->metric + strlen(stubname)); } if (!(pmode & PMODE_STUB) && r->dests->next != NULL) { destinations *dn = r->dests->next; fprintf(f, " send to"); if (dn->next == NULL) { fprintf(f, " %s\n", dn->cl->name); } else { for (; dn != NULL; dn = dn->next) fprintf(f, "\n %s", dn->cl->name); fprintf(f, "\n"); } } fprintf(f, "%s ;\n", r->stop ? " stop\n" : ""); } else if (r->dests->cl->type == REWRITE) { fprintf(f, "rewrite %s\n into %s\n ;\n", r->pattern, r->dests->cl->members.replacement); } else if (r->dests->cl->type == GROUP) { size_t cnt = 0; route *rwalk; char blockname[64]; char *b = &blockname[sizeof(blockname) - 1]; char *p; for (rwalk = r->dests->cl->members.routes; rwalk != NULL; rwalk = rwalk->next) cnt++; /* reverse the name, to make it human consumable */ *b-- ='\0'; for (p = r->dests->cl->name; *p != '\0' && b > blockname; p++) *b-- = *p; fprintf(f, "# common pattern group '%s' " "contains %zu aggregations/matches\n", ++b, cnt); if (mode & PMODE_AGGR) { router srtr; memset(&srtr, 0, sizeof(srtr)); srtr.routes = r->dests->cl->members.routes; /* recurse */ router_printconfig(&srtr, f, pmode); } } else if (r->dests->cl->type == AGGRSTUB || r->dests->cl->type == STATSTUB) { if (pmode & PMODE_STUB) { fprintf(f, "# stub match for aggregate/statistics rule " "with send to\n"); fprintf(f, "match ^%s\n send to", r->pattern); if (r->dests->cl->members.routes->dests->next == NULL) { fprintf(f, " %s", r->dests->cl->members.routes->dests->cl->name); } else { destinations *d = r->dests->cl->members.routes->dests; for (; d != NULL; d = d->next) fprintf(f, "\n %s", d->cl->name); } fprintf(f, "%s\n ;\n", r->stop ? "\n stop" : ""); } else if (r->dests->cl->type == STATSTUB) { destinations *d = r->dests->cl->members.routes->dests; fprintf(f, "send statistics to"); if (d->next == NULL) { fprintf(f, " %s", d->cl->name); } else { for ( ; d != NULL; d = d->next) fprintf(f, "\n %s", d->cl->name); } fprintf(f, "\n stop\n ;\n"); } } else { route *or = r; destinations *d; fprintf(f, "match"); if (r->next == NULL || r->next->dests != or->dests) { fprintf(f, " %s\n", r->matchtype == MATCHALL ? "*" : r->pattern); } else { fprintf(f, "\n"); do { fprintf(f, " %s\n", r->matchtype == MATCHALL ? "*" : r->pattern); } while (r->next != NULL && r->next->dests == or->dests && (r = r->next) != NULL); } d = or->dests; if (d->cl->type == VALIDATION) { validate *v = d->cl->members.validation; fprintf(f, " validate %s else %s\n", v->rule->pattern, v->action == VAL_LOG ? "log" : "drop"); /* hide this pseudo target */ d = d->next; } if (d != NULL) { fprintf(f, " send to"); if (d->next == NULL) { fprintf(f, " %s", d->cl->name); } else { for ( ; d != NULL; d = d->next) fprintf(f, "\n %s", d->cl->name); } fprintf(f, "\n%s ;\n", or->stop ? " stop\n" : ""); } else { fprintf(f, " ;\n"); } } } fflush(f); } /** * Prints the differences between old and new at the highest level of * the change. (When a cluster is gone, its servers are not * considered.) Returns whether there were changes found. */ char router_printdiffs(router *old, router *new, FILE *out) { FILE *f; int fd; char *tmp = getenv("TMPDIR"); char patho[512]; char pathn[512]; char buf[1024]; size_t len; int ret; mode_t mask; /* First idea was to printconfig both old and new, and run diff -u * on them. Simple and effective. Does require diff and two files * though. For now, I guess we can live with that. */ if (tmp == NULL) tmp = "/tmp"; snprintf(patho, sizeof(patho), "%s/carbon-c-relay_route.XXXXXX", tmp); mask = umask(S_IWGRP | S_IWOTH); fd = mkstemp(patho); umask(mask); if (fd < 0) { logerr("failed to create temporary file: %s\n", strerror(errno)); return 1; } f = fdopen(fd, "r+"); if (f == NULL) { logerr("failed to open stream for file '%s': %s\n", patho, strerror(errno)); return 1; } router_printconfig(old, f, PMODE_AGGR); fclose(f); snprintf(pathn, sizeof(pathn), "%s/carbon-c-relay_route.XXXXXX", tmp); mask = umask(S_IWGRP | S_IWOTH); fd = mkstemp(pathn); umask(mask); if (fd < 0) { logerr("failed to create temporary file: %s\n", strerror(errno)); return 1; } f = fdopen(fd, "r+"); if (f == NULL) { logerr("failed to open stream for file '%s': %s\n", pathn, strerror(errno)); return 1; } router_printconfig(new, f, PMODE_AGGR); fclose(f); /* diff and print its output */ snprintf(buf, sizeof(buf), "diff -u %s %s", patho, pathn); f = popen(buf, "r"); while ((len = fread(buf, 1, sizeof(buf) - 1, f)) > 0) { if (fwrite(buf, len, 1, out) == 0) /* ignore */ ; } ret = pclose(f); unlink(patho); unlink(pathn); /* ret: * 0 - no diffs * 1 - diffs * 2 - some error (file not found?) * let's say that an error means potential difference */ return ret != 0; } /** * Evaluates all servers in new and if an identical server exists in * old, swaps their queues. */ void router_transplant_queues(router *new, router *old) { servers *os; servers *ns; for (ns = new->srvrs; ns != NULL; ns = ns->next) { for (os = old->srvrs; os != NULL; os = os->next) { if (strcmp(server_ip(ns->server), server_ip(os->server)) == 0 && server_port(ns->server) == server_port(os->server) && server_ctype(ns->server) == server_ctype(os->server)) { server_swap_queue(ns->server, os->server); continue; } } } } /** * Starts all resources (servers) associated to this router. */ char router_start(router *rtr) { servers *s; char ret = 0; int err; for (s = rtr->srvrs; s != NULL; s = s->next) { if ((err = server_start(s->server)) != 0) { logerr("failed to start server %s:%u: %s\n", server_ip(s->server), server_port(s->server), strerror(err)); ret = 1; } } return ret; } /** * Shuts down all resources (servers) associated to this router. */ void router_shutdown(router *rtr) { servers *s; for (s = rtr->srvrs; s != NULL; s = s->next) server_shutdown(s->server); } /** * Free the routes and all associated resources. */ void router_free(router *rtr) { servers *s; router_free_intern(rtr->clusters, rtr->routes); /* free all servers from the pool, in case of secondaries, the * previous call to router_shutdown made sure nothing references the * servers anymore */ for (s = rtr->srvrs; s != NULL; s = s->next) server_free(s->server); ra_free(rtr); free(rtr); } inline static char router_metric_matches( const route *r, char *metric, char *firstspace, regmatch_t *pmatch) { char ret = 0; char firstspc = *firstspace; switch (r->matchtype) { case MATCHALL: ret = 1; break; case REGEX: *firstspace = '\0'; ret = regexec(&r->rule, metric, r->nmatch, pmatch, 0) == 0; *firstspace = firstspc; break; case CONTAINS: *firstspace = '\0'; ret = strstr(metric, r->strmatch) != NULL; *firstspace = firstspc; break; case STARTS_WITH: ret = strncmp(metric, r->strmatch, strlen(r->strmatch)) == 0; break; case ENDS_WITH: *firstspace = '\0'; ret = strcmp( firstspace - strlen(r->strmatch), r->strmatch) == 0; *firstspace = firstspc; break; case MATCHES: *firstspace = '\0'; ret = strcmp(metric, r->strmatch) == 0; *firstspace = firstspc; break; default: ret = 0; break; } return ret; } inline size_t router_rewrite_metric( char (*newmetric)[METRIC_BUFSIZ], char **newfirstspace, const char *metric, const char *firstspace, const char *replacement, const size_t nmatch, const regmatch_t *pmatch) { char escape = 0; int ref = 0; char *s = *newmetric; const char *p; const char *q; const char *t; enum rewrite_case { RETAIN, LOWER, UPPER } rcase = RETAIN; assert(pmatch != NULL); /* insert leading part */ q = metric; t = metric + pmatch[0].rm_so; if (s - *newmetric + t - q < sizeof(*newmetric)) { while (q < t) *s++ = *q++; } else { return 0; /* won't fit, don't try further */ } for (p = replacement; ; p++) { switch (*p) { case '\\': if (!escape) { escape = 1; rcase = RETAIN; break; } /* fall through so we handle \1\2 */ default: if (escape == 1 && rcase == RETAIN && *p == '_') { rcase = LOWER; } else if (escape == 1 && rcase == RETAIN && *p == '^') { rcase = UPPER; } else if (escape && *p >= '0' && *p <= '9') { escape = 2; ref *= 10; ref += *p - '0'; } else { if (escape) { if (ref > 0 && ref <= nmatch && pmatch[ref].rm_so >= 0) { /* insert match part */ q = metric + pmatch[ref].rm_so; t = metric + pmatch[ref].rm_eo; if (s - *newmetric + t - q < sizeof(*newmetric)) { switch (rcase) { case RETAIN: while (q < t) *s++ = *q++; break; case LOWER: while (q < t) *s++ = (char)tolower(*q++); break; case UPPER: while (q < t) *s++ = (char)toupper(*q++); break; } } } ref = 0; } if (*p != '\\') { /* \1\2 case */ escape = 0; rcase = RETAIN; if (s - *newmetric + 1 < sizeof(*newmetric)) *s++ = *p; } } break; } if (*p == '\0') break; } /* undo trailing \0 */ s--; /* insert remaining part */ q = metric + pmatch[0].rm_eo; t = firstspace; if (s - *newmetric + t - q < sizeof(*newmetric)) { while (q < t) *s++ = *q++; } else { return 0; /* won't fit, don't try further */ } /* record new position of firstspace */ *newfirstspace = s; /* copy data part */ if (s - *newmetric + strlen(firstspace) < sizeof(*newmetric)) { for (p = firstspace; *p != '\0'; p++) *s++ = *p; *s++ = '\0'; return s - *newmetric; } return 0; /* we couldn't copy everything */ } static char router_route_intern( char *blackholed, destination ret[], size_t *curlen, size_t retsize, char *srcaddr, char *metric, char *firstspace, const route *r) { const route *w; destinations *d; char stop = 0; char wassent = 0; const char *p; const char *q = NULL; /* pacify compiler, won't happen in reality */ const char *t; char newmetric[METRIC_BUFSIZ]; char *newfirstspace = NULL; size_t len; regmatch_t pmatch[RE_MAX_MATCHES]; #define failif(RETLEN, WANTLEN) \ if (WANTLEN > RETLEN) { \ logerr("router_route: out of destination slots, " \ "increase CONN_DESTS_SIZE in router.h\n"); \ return 1; \ } for (w = r; w != NULL; w = w->next) { if (w->dests->cl->type == GROUP) { /* strrstr doesn't exist, grrr * therefore the pattern in the group is stored in reverse, * such that we can start matching the tail easily without * having to calculate the end of the pattern string all the * time */ for (p = firstspace - 1; p >= metric; p--) { for (q = w->dests->cl->name, t = p; *q != '\0' && t >= metric; q++, t--) { if (*q != *t) break; } if (*q == '\0') break; } /* indirection */ assert(q != NULL); if (*q == '\0') stop = router_route_intern( blackholed, ret, curlen, retsize, srcaddr, metric, firstspace, w->dests->cl->members.routes); } else if (router_metric_matches(w, metric, firstspace, pmatch)) { stop = w->stop; /* rule matches, send to destination(s) */ for (d = w->dests; d != NULL; d = d->next) { switch (d->cl->type) { case BLACKHOLE: *blackholed = 1; break; case FILELOGIP: { servers *s; snprintf(newmetric, sizeof(newmetric), "%s %s", srcaddr, metric); for (s = d->cl->members.forward; s != NULL; s = s->next) { failif(retsize, *curlen + 1); ret[*curlen].dest = s->server; ret[(*curlen)++].metric = strdup(newmetric); } wassent = 1; } break; case FILELOG: case FORWARD: { /* simple case, no logic necessary */ servers *s; for (s = d->cl->members.forward; s != NULL; s = s->next) { failif(retsize, *curlen + 1); ret[*curlen].dest = s->server; ret[(*curlen)++].metric = strdup(metric); } wassent = 1; } break; case ANYOF: { /* we queue the same metrics at the same server */ unsigned int hash; fnv1a_32(hash, p, metric, firstspace); /* We could use the retry approach here, but since * our c is very small compared to MAX_INT, the bias * we introduce for the last few of the range * (MAX_INT % c) can be considered neglicible given * the number of occurances of c in the range of * MAX_INT, therefore we stick with a simple mod. */ hash %= d->cl->members.anyof->count; failif(retsize, *curlen + 1); ret[*curlen].dest = d->cl->members.anyof->servers[hash]; ret[(*curlen)++].metric = strdup(metric); wassent = 1; } break; case FAILOVER: { /* queue at the first non-failing server */ unsigned short i; failif(retsize, *curlen + 1); ret[*curlen].dest = NULL; for (i = 0; i < d->cl->members.anyof->count; i++) { server *s = d->cl->members.anyof->servers[i]; if (server_failed(s)) continue; ret[*curlen].dest = s; break; } if (ret[*curlen].dest == NULL) /* all failed, take first server */ ret[*curlen].dest = d->cl->members.anyof->servers[0]; ret[(*curlen)++].metric = strdup(metric); wassent = 1; } break; case CARBON_CH: case FNV1A_CH: case JUMP_CH: { /* let the ring(bearer) decide */ failif(retsize, *curlen + d->cl->members.ch->repl_factor); ch_get_nodes( &ret[*curlen], d->cl->members.ch->ring, d->cl->members.ch->repl_factor, metric, firstspace); *curlen += d->cl->members.ch->repl_factor; wassent = 1; } break; case AGGREGATION: { /* aggregation rule */ aggregator_putmetric( d->cl->members.aggregation, metric, firstspace, w->nmatch, pmatch); wassent = 1; /* we need to break out of the inner loop. since * the rest of dests are meant for the stub, and * we should certainly not process it now */ while (d->next != NULL) d = d->next; } break; case REWRITE: { /* rewrite metric name */ if ((len = router_rewrite_metric( &newmetric, &newfirstspace, metric, firstspace, d->cl->members.replacement, w->nmatch, pmatch)) == 0) { logerr("router_route: failed to rewrite " "metric: newmetric size too small to hold " "replacement (%s -> %s)\n", metric, d->cl->members.replacement); break; }; /* scary! write back the rewritten metric */ memcpy(metric, newmetric, len); firstspace = metric + (newfirstspace - newmetric); } break; case AGGRSTUB: case STATSTUB: { /* strip off the stub pattern, and reroute this * thing */ router_route_intern( blackholed, ret, curlen, retsize, srcaddr, metric + strlen(w->pattern), firstspace + strlen(w->pattern), w->dests->cl->members.routes); } break; case VALIDATION: { /* test whether data matches, if not, either log * or drop and stop */ char *lastchr = firstspace + strlen(firstspace) - 1; if (router_metric_matches( w->dests->cl->members.validation->rule, firstspace + 1, lastchr, pmatch)) break; if (w->dests->cl->members.validation->action == VAL_LOG) { logerr("dropping metric due to validation error: " "%s", metric); wassent = 1; } /* only stop if this is a validate without * destinations */ stop |= d->next == NULL; /* break out of the dests loop */ while (d->next != NULL) d = d->next; break; } case GROUP: { /* this should not happen */ } break; } } /* stop processing further rules if requested */ if (stop) break; } } if (!wassent) *blackholed = 1; return stop; } /** * Looks up the locations the given metric_path should be sent to, and * returns the list of servers in ret, the number of servers is * returned in retcnt. * Returns whether the metric was blackholed (e.g. not routed anywhere). */ inline char router_route( router *rtr, destination ret[], size_t *retcnt, size_t retsize, char *srcaddr, char *metric, char *firstspace) { size_t curlen = 0; char blackholed = 0; (void)router_route_intern(&blackholed, ret, &curlen, retsize, srcaddr, metric, firstspace, rtr->routes); *retcnt = curlen; return blackholed; } /** * Prints for metric_path which rules and/or aggregations would be * triggered. Useful for testing regular expressions. */ static char router_test_intern(char *metric, char *firstspace, route *routes) { route *w; destinations *d; char stop = 0; char gotmatch = 0; char newmetric[METRIC_BUFSIZ]; char *newfirstspace = NULL; size_t len; regmatch_t pmatch[RE_MAX_MATCHES]; for (w = routes; w != NULL; w = w->next) { if (w->dests->cl->type == GROUP) { /* just recurse, in test mode performance shouldn't be an * issue at all */ gotmatch |= router_test_intern( metric, firstspace, w->dests->cl->members.routes); if (gotmatch & 2) break; } else if (router_metric_matches(w, metric, firstspace, pmatch)) { gotmatch = 1; switch (w->dests->cl->type) { case AGGREGATION: fprintf(stdout, "aggregation\n"); break; case REWRITE: fprintf(stdout, "rewrite\n"); break; case AGGRSTUB: case STATSTUB: { gotmatch |= router_test_intern( metric + strlen(w->pattern), firstspace + strlen(w->pattern), w->dests->cl->members.routes); return gotmatch; } break; default: fprintf(stdout, "match\n"); break; } *firstspace = '\0'; switch (w->matchtype) { case MATCHALL: fprintf(stdout, " * -> %s\n", metric); break; case REGEX: fprintf(stdout, " %s (regex) -> %s\n", w->pattern, metric); break; default: { char *x; switch (w->matchtype) { case CONTAINS: x = "strstr"; break; case STARTS_WITH: x = "strncmp"; break; case ENDS_WITH: x = "tailcmp"; break; case MATCHES: x = "strcmp"; break; default: x = "!impossible?"; break; } fprintf(stdout, " %s [%s: %s]\n -> %s\n", w->pattern, x, w->strmatch, metric); } break; } *firstspace = ' '; stop = w->stop; for (d = w->dests; d != NULL; d = d->next) { switch (d->cl->type) { case AGGREGATION: { struct _aggr_computes *ac; int stublen = 0; char percentile[16]; if (mode & MODE_DEBUG || d->next == NULL) { stublen = 0; } else { char x; stublen = snprintf(&x, 1, STUB_AGGR "%p__", d->cl->members.aggregation); } for (ac = d->cl->members.aggregation->computes; ac != NULL; ac = ac->next) { if (w->nmatch == 0 || (len = router_rewrite_metric( &newmetric, &newfirstspace, metric, firstspace, ac->metric, w->nmatch, pmatch)) == 0) { if (w->nmatch > 0) { fprintf(stderr, "router_test: failed to " "rewrite metric: newmetric size too " "small to hold replacement " "(%s -> %s)\n", metric, ac->metric); break; } len = snprintf(newmetric, sizeof(newmetric), "%s", ac->metric); if (len >= sizeof(newmetric)) len = sizeof(newmetric) - 1; newfirstspace = newmetric + len; } snprintf(percentile, sizeof(percentile), "percentile%d", ac->percentile); fprintf(stdout, " %s%s%s%s -> %s\n", ac->type == SUM ? "sum" : ac->type == CNT ? "count" : ac->type == MAX ? "max" : ac->type == MIN ? "min" : ac->type == AVG ? "average" : ac->type == MEDN ? "median" : ac->type == PCTL ? percentile : ac->type == VAR ? "variance" : ac->type == SDEV ? "stddev" : "", w->nmatch > 0 ? "(" : "", w->nmatch > 0 ? ac->metric + stublen : "", w->nmatch > 0 ? ")" : "", newmetric + stublen); if (mode & MODE_DEBUG && d->next != NULL) { gotmatch |= router_test_intern( newmetric, newfirstspace, routes); } } if (mode & MODE_DEBUG) { return gotmatch; } else { gotmatch |= 4; } } break; case BLACKHOLE: { fprintf(stdout, " blackholed\n"); } break; case REWRITE: { /* rewrite metric name */ if ((len = router_rewrite_metric( &newmetric, &newfirstspace, metric, firstspace, d->cl->members.replacement, w->nmatch, pmatch)) == 0) { fprintf(stderr, "router_test: failed to rewrite " "metric: newmetric size too small to hold " "replacement (%s -> %s)\n", metric, d->cl->members.replacement); break; }; /* scary! write back the rewritten metric */ memcpy(metric, newmetric, len); firstspace = metric + (newfirstspace - newmetric); *firstspace = '\0'; fprintf(stdout, " into(%s) -> %s\n", d->cl->members.replacement, metric); *firstspace = ' '; } break; case FORWARD: { servers *s; fprintf(stdout, " forward(%s)\n", d->cl->name); for (s = d->cl->members.forward; s != NULL; s = s->next) fprintf(stdout, " %s:%d\n", server_ip(s->server), server_port(s->server)); } break; case CARBON_CH: case FNV1A_CH: case JUMP_CH: { destination dst[CONN_DESTS_SIZE]; int i; fprintf(stdout, " %s_ch(%s)\n", d->cl->type == CARBON_CH ? "carbon" : d->cl->type == FNV1A_CH ? "fnv1a" : "jump_fnv1a", d->cl->name); if (gotmatch & 4) break; if (mode & MODE_DEBUG) { fprintf(stdout, " hash_pos(%d)\n", ch_gethashpos(d->cl->members.ch->ring, metric, firstspace)); } ch_get_nodes(dst, d->cl->members.ch->ring, d->cl->members.ch->repl_factor, metric, firstspace); for (i = 0; i < d->cl->members.ch->repl_factor; i++) { fprintf(stdout, " %s:%d\n", server_ip(dst[i].dest), server_port(dst[i].dest)); free((char *)dst[i].metric); } } break; case FAILOVER: case ANYOF: { unsigned int hash; fprintf(stdout, " %s(%s)\n", d->cl->type == ANYOF ? "any_of" : "failover", d->cl->name); if (gotmatch & 4) break; if (d->cl->type == ANYOF) { const char *p; fnv1a_32(hash, p, metric, firstspace); hash %= d->cl->members.anyof->count; } else { hash = 0; } fprintf(stdout, " %s:%d\n", server_ip(d->cl->members.anyof->servers[hash]), server_port(d->cl->members.anyof->servers[hash])); } break; case VALIDATION: { char *lastspc = firstspace + strlen(firstspace); fprintf(stdout, " validate\n %s -> %s\n", d->cl->members.validation->rule->pattern, firstspace + 1); if (router_metric_matches( d->cl->members.validation->rule, firstspace + 1, lastspc, pmatch)) { fprintf(stdout, " match\n"); } else { fprintf(stdout, " fail -> %s\n", d->cl->members.validation->action == VAL_LOG ? "log" : "drop"); stop |= d->next == NULL; while (d->next != NULL) d = d->next; } } break; default: { fprintf(stdout, " cluster(%s)\n", d->cl->name); } break; } } if (stop) { gotmatch = 3; fprintf(stdout, " stop\n"); break; } } } return gotmatch; } void router_test(router *rtr, char *metric) { char *firstspace; for (firstspace = metric; *firstspace != '\0'; firstspace++) if (*firstspace == ' ') break; if (!router_test_intern(metric, firstspace, rtr->routes)) { *firstspace = '\0'; fprintf(stdout, "nothing matched %s\n", metric); } fflush(stdout); } carbon-c-relay-2.5/router.h000066400000000000000000000037631303466152100156610ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef ROUTER_H #define ROUTER_H 1 #include #include #include "server.h" #include "aggregator.h" #define PMODE_NORM (1 << 0) #define PMODE_AGGR (1 << 1) #define PMODE_HASH (1 << 2) #define PMODE_STUB (1 << 3) #define PMODE_DEBUG (PMODE_HASH | PMODE_STUB) #define CONN_DESTS_SIZE 64 typedef struct { const char *metric; server *dest; } destination; typedef struct _router router; #define RE_MAX_MATCHES 64 router *router_readconfig(router *orig, const char *path, size_t queuesize, size_t batchsize, int maxstalls, unsigned short iotimeout, unsigned int sockbufsize); void router_optimise(router *r); char router_printdiffs(router *old, router *new, FILE *out); void router_transplant_queues(router *new, router *old); char router_start(router *r); size_t router_rewrite_metric(char (*newmetric)[METRIC_BUFSIZ], char **newfirstspace, const char *metric, const char *firstspace, const char *replacement, const size_t nmatch, const regmatch_t *pmatch); void router_printconfig(router *r, FILE *f, char mode); char router_route(router *r, destination ret[], size_t *retcnt, size_t retsize, char *srcaddr, char *metric, char *firstspace); void router_test(router *r, char *metric_path); server **router_getservers(router *r); aggregator *router_getaggregators(router *r); char *router_getcollectorstub(router *r); void router_shutdown(router *r); void router_free(router *r); #endif carbon-c-relay-2.5/server.c000066400000000000000000000566301303466152100156430ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "relay.h" #include "queue.h" #include "dispatcher.h" #include "collector.h" #include "server.h" struct _server { const char *ip; unsigned short port; char *instance; struct addrinfo *saddr; int fd; queue *queue; size_t bsize; short iotimeout; unsigned int sockbufsize; unsigned char maxstalls:SERVER_STALL_BITS; const char **batch; serv_ctype ctype; pthread_t tid; struct _server **secondaries; size_t secondariescnt; char failover:1; char failure; /* full byte for atomic access */ char running:1; char keep_running; /* full byte for atomic access */ unsigned char stallseq:SERVER_STALL_BITS; size_t metrics; size_t dropped; size_t stalls; size_t ticks; size_t prevmetrics; size_t prevdropped; size_t prevstalls; size_t prevticks; }; /** * Reads from the queue and sends items to the remote server. This * function is designed to be a thread. Data sending is attempted to be * batched, but sent one by one to reduce loss on sending failure. * A connection with the server is maintained for as long as there is * data to be written. As soon as there is none, the connection is * dropped if a timeout of DISCONNECT_WAIT_TIME exceeds. */ static void * server_queuereader(void *d) { server *self = (server *)d; size_t len; ssize_t slen; const char **metric = self->batch; struct timeval start, stop; struct timeval timeout; queue *squeue; char idle = 0; size_t *secpos = NULL; unsigned char cnt; const char *p; *metric = NULL; #define FAIL_WAIT_TIME 6 /* 6 * 250ms = 1.5s */ #define DISCONNECT_WAIT_TIME 12 /* 12 * 250ms = 3s */ #define LEN_CRITICAL(Q) (queue_free(Q) < self->bsize) self->running = 1; while (1) { if (queue_len(self->queue) == 0) { /* if we're idling, close the TCP connection, this allows us * to reduce connections, while keeping the connection alive * if we're writing a lot */ gettimeofday(&start, NULL); if (self->ctype == CON_TCP && self->fd >= 0 && idle++ > DISCONNECT_WAIT_TIME) { close(self->fd); self->fd = -1; } gettimeofday(&stop, NULL); __sync_add_and_fetch(&(self->ticks), timediff(start, stop)); if (__sync_bool_compare_and_swap(&(self->keep_running), 0, 0)) break; /* nothing to do, so slow down for a bit */ usleep((200 + (rand() % 100)) * 1000); /* 200ms - 300ms */ /* if we are in failure mode, keep checking if we can * connect, this avoids unnecessary queue moves */ if (__sync_bool_compare_and_swap(&(self->failure), 0, 0)) /* it makes no sense to try and do something, so skip */ continue; } else if (self->secondariescnt > 0 && (__sync_add_and_fetch(&(self->failure), 0) >= FAIL_WAIT_TIME || (!self->failover && LEN_CRITICAL(self->queue)))) { size_t i; gettimeofday(&start, NULL); if (self->secondariescnt > 0) { if (secpos == NULL) { secpos = malloc(sizeof(size_t) * self->secondariescnt); if (secpos == NULL) { logerr("server: failed to allocate memory " "for secpos\n"); gettimeofday(&stop, NULL); __sync_add_and_fetch(&(self->ticks), timediff(start, stop)); continue; } for (i = 0; i < self->secondariescnt; i++) secpos[i] = i; } if (!self->failover) { /* randomise the failover list such that in the * grand scheme of things we don't punish the first * working server in the list to deal with all * traffic meant for a now failing server */ for (i = 0; i < self->secondariescnt; i++) { size_t n = rand() % (self->secondariescnt - i); if (n != i) { size_t t = secpos[n]; secpos[n] = secpos[i]; secpos[i] = t; } } } } /* offload data from our queue to our secondaries * when doing so, observe the following: * - avoid nodes that are in failure mode * - avoid nodes which queues are >= critical_len * when no nodes remain given the above * - send to nodes which queue size < critical_len * where there are no such nodes * - do nothing (we will overflow, since we can't send * anywhere) */ *metric = NULL; squeue = NULL; for (i = 0; i < self->secondariescnt; i++) { /* both conditions below make sure we skip ourself */ if (__sync_add_and_fetch( &(self->secondaries[secpos[i]]->failure), 0)) continue; squeue = self->secondaries[secpos[i]]->queue; if (!self->failover && LEN_CRITICAL(squeue)) { squeue = NULL; continue; } if (*metric == NULL) { /* send up to batch size of our queue to this queue */ len = queue_dequeue_vector( self->batch, self->queue, self->bsize); self->batch[len] = NULL; metric = self->batch; } for (; *metric != NULL; metric++) if (!queue_putback(squeue, *metric)) break; /* try to put back stuff that didn't fit */ for (; *metric != NULL; metric++) if (!queue_putback(self->queue, *metric)) break; } for (; *metric != NULL; metric++) { if (mode & MODE_DEBUG) logerr("dropping metric: %s", *metric); free((char *)*metric); __sync_add_and_fetch(&(self->dropped), 1); } gettimeofday(&stop, NULL); self->ticks += timediff(start, stop); if (squeue == NULL) { /* we couldn't do anything, take it easy for a bit */ if (__sync_add_and_fetch(&(self->failure), 0) > 1) { /* This is a compound because I can't seem to figure * out how to atomically just "set" a variable. * It's not bad when in the middle there is a ++, * all that counts is that afterwards its > 0. */ __sync_and_and_fetch(&(self->failure), 0); __sync_add_and_fetch(&(self->failure), 1); } if (__sync_bool_compare_and_swap(&(self->keep_running), 0, 0)) break; usleep((200 + (rand() % 100)) * 1000); /* 200ms - 300ms */ } } else if (__sync_add_and_fetch(&(self->failure), 0) > 0) { if (__sync_bool_compare_and_swap(&(self->keep_running), 0, 0)) break; usleep((200 + (rand() % 100)) * 1000); /* 200ms - 300ms */ /* avoid overflowing */ if (__sync_add_and_fetch(&(self->failure), 0) > FAIL_WAIT_TIME) __sync_sub_and_fetch(&(self->failure), 1); } /* at this point we've got work to do, if we're instructed to * shut down, however, try to get everything out of the door * (until we fail, see top of this loop) */ gettimeofday(&start, NULL); /* try to connect */ if (self->fd < 0) { if (self->ctype == CON_PIPE) { int intconn[2]; if (pipe(intconn) < 0) { if (__sync_fetch_and_add(&(self->failure), 1) == 0) logerr("failed to create pipe: %s\n", strerror(errno)); continue; } dispatch_addconnection(intconn[0]); self->fd = intconn[1]; } else if (self->ctype == CON_UDP) { if ((self->fd = socket(self->saddr->ai_family, self->saddr->ai_socktype, self->saddr->ai_protocol)) < 0) { if (__sync_fetch_and_add(&(self->failure), 1) == 0) logerr("failed to create udp socket: %s\n", strerror(errno)); continue; } if (connect(self->fd, self->saddr->ai_addr, self->saddr->ai_addrlen) < 0) { if (__sync_fetch_and_add(&(self->failure), 1) == 0) logerr("failed to connect udp socket: %s\n", strerror(errno)); close(self->fd); self->fd = -1; continue; } } else if (self->ctype == CON_FILE) { if ((self->fd = open(self->ip, O_WRONLY | O_APPEND | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)) < 0) { if (__sync_fetch_and_add(&(self->failure), 1) == 0) logerr("failed to open file '%s': %s\n", self->ip, strerror(errno)); continue; } } else { int ret; int args; if ((self->fd = socket(self->saddr->ai_family, self->saddr->ai_socktype, self->saddr->ai_protocol)) < 0) { if (__sync_fetch_and_add(&(self->failure), 1) == 0) logerr("failed to create socket: %s\n", strerror(errno)); continue; } /* put socket in non-blocking mode such that we can * poll() (time-out) on the connect() call */ args = fcntl(self->fd, F_GETFL, NULL); if (fcntl(self->fd, F_SETFL, args | O_NONBLOCK) < 0) { logerr("failed to set socket non-blocking mode: %s\n", strerror(errno)); close(self->fd); self->fd = -1; continue; } ret = connect(self->fd, self->saddr->ai_addr, self->saddr->ai_addrlen); if (ret < 0 && errno == EINPROGRESS) { /* wait for connection to succeed if the OS thinks * it can succeed */ struct pollfd ufds[1]; ufds[0].fd = self->fd; ufds[0].events = POLLIN | POLLOUT; ret = poll(ufds, 1, self->iotimeout + (rand() % 100)); if (ret == 0) { /* time limit expired */ if (__sync_fetch_and_add(&(self->failure), 1) == 0) logerr("failed to connect() to " "%s:%u: Operation timed out\n", self->ip, self->port); close(self->fd); self->fd = -1; continue; } else if (ret < 0) { /* some select error occurred */ if (__sync_fetch_and_add(&(self->failure), 1) == 0) logerr("failed to poll() for %s:%u: %s\n", self->ip, self->port, strerror(errno)); close(self->fd); self->fd = -1; continue; } else { if (ufds[0].revents & POLLHUP) { if (__sync_fetch_and_add(&(self->failure), 1) == 0) logerr("failed to connect() for %s:%u: " "Hangup\n", self->ip, self->port); close(self->fd); self->fd = -1; continue; } } } else if (ret < 0) { if (__sync_fetch_and_add(&(self->failure), 1) == 0) { logerr("failed to connect() to %s:%u: %s\n", self->ip, self->port, strerror(errno)); dispatch_check_rlimit_and_warn(); } close(self->fd); self->fd = -1; continue; } /* make socket blocking again */ if (fcntl(self->fd, F_SETFL, args) < 0) { logerr("failed to remove socket non-blocking mode: %s\n", strerror(errno)); close(self->fd); self->fd = -1; continue; } /* disable Nagle's algorithm, issue #208 */ args = 1; if (setsockopt(self->fd, IPPROTO_TCP, TCP_NODELAY, &args, sizeof(args)) != 0) ; /* ignore */ } /* ensure we will break out of connections being stuck more * quickly than the kernel would give up */ timeout.tv_sec = 10; timeout.tv_usec = (rand() % 300) * 1000; if (setsockopt(self->fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)) != 0) ; /* ignore */ if (self->sockbufsize > 0) if (setsockopt(self->fd, SOL_SOCKET, SO_SNDBUF, &self->sockbufsize, sizeof(self->sockbufsize)) != 0) ; /* ignore */ #ifdef SO_NOSIGPIPE if (self->ctype == CON_TCP || self->ctype == CON_UDP) if (setsockopt(self->fd, SOL_SOCKET, SO_NOSIGPIPE, NULL, 0) != 0) logout("warning: failed to ignore SIGPIPE on socket: %s\n", strerror(errno)); #endif } /* send up to batch size */ len = queue_dequeue_vector(self->batch, self->queue, self->bsize); self->batch[len] = NULL; metric = self->batch; if (len != 0 && __sync_bool_compare_and_swap(&(self->keep_running), 0, 0)) { /* be noisy during shutdown so we can track any slowing down * servers, possibly preventing us to shut down */ logerr("shutting down %s:%u: waiting for %zu metrics\n", self->ip, self->port, len + queue_len(self->queue)); } if (len == 0 && __sync_add_and_fetch(&(self->failure), 0)) { /* if we don't have anything to send, we have at least a * connection succeed, so assume the server is up again, * this is in particular important for recovering this * node by probes, to avoid starvation of this server since * its queue is possibly being offloaded to secondaries */ if (self->ctype != CON_UDP) logerr("server %s:%u: OK after probe\n", self->ip, self->port); __sync_and_and_fetch(&(self->failure), 0); } for (; *metric != NULL; metric++) { len = strlen(*metric); /* Write to the stream, this may not succeed completely due * to flow control and whatnot, which the docs suggest need * resuming to complete. So, use a loop, but to avoid * getting endlessly stuck on this, only try a limited * number of times for a single metric. */ for (cnt = 0, p = *metric; cnt < 10 && (slen = write(self->fd, p, len)) != len && slen >= 0; cnt++) { p += slen; len -= slen; } if (slen != len) { /* not fully sent (after tries), or failure * close connection regardless so we don't get * synchonisation problems */ if (self->ctype != CON_UDP && __sync_fetch_and_add(&(self->failure), 1) == 0) logerr("failed to write() to %s:%u: %s\n", self->ip, self->port, (slen < 0 ? strerror(errno) : "incomplete write")); close(self->fd); self->fd = -1; /* put back stuff we couldn't process */ for (; *metric != NULL; metric++) { if (!queue_putback(self->queue, *metric)) { if (mode & MODE_DEBUG) logerr("server %s:%u: dropping metric: %s", self->ip, self->port, *metric); free((char *)*metric); __sync_add_and_fetch(&(self->dropped), 1); } } break; } else if (!__sync_bool_compare_and_swap(&(self->failure), 0, 0)) { if (self->ctype != CON_UDP) logerr("server %s:%u: OK\n", self->ip, self->port); __sync_and_and_fetch(&(self->failure), 0); } free((char *)*metric); __sync_add_and_fetch(&(self->metrics), 1); } gettimeofday(&stop, NULL); __sync_add_and_fetch(&(self->ticks), timediff(start, stop)); idle = 0; } self->running = 0; if (self->fd >= 0) close(self->fd); if (secpos != NULL) free(secpos); return NULL; } /** * Allocate a new (outbound) server. Effectively this means a thread * that reads from the queue and sends this as good as it can to the ip * address and port associated. */ server * server_new( const char *ip, unsigned short port, serv_ctype ctype, struct addrinfo *saddr, size_t qsize, size_t bsize, int maxstalls, unsigned short iotimeout, unsigned int sockbufsize) { server *ret; if ((ret = malloc(sizeof(server))) == NULL) return NULL; ret->ctype = ctype; ret->tid = 0; ret->secondaries = NULL; ret->secondariescnt = 0; ret->ip = strdup(ip); if (ret->ip == NULL) { free(ret); return NULL; } ret->port = port; ret->instance = NULL; ret->bsize = bsize; ret->iotimeout = iotimeout < 250 ? 600 : iotimeout; ret->sockbufsize = sockbufsize; ret->maxstalls = maxstalls; if ((ret->batch = malloc(sizeof(char *) * (bsize + 1))) == NULL) { free((char *)ret->ip); free(ret); return NULL; } ret->fd = -1; ret->saddr = saddr; ret->queue = queue_new(qsize); if (ret->queue == NULL) { free(ret->batch); free((char *)ret->ip); free(ret); return NULL; } ret->failover = 0; ret->failure = 0; ret->running = 0; ret->keep_running = 1; ret->stallseq = 0; ret->metrics = 0; ret->dropped = 0; ret->stalls = 0; ret->ticks = 0; ret->prevmetrics = 0; ret->prevdropped = 0; ret->prevstalls = 0; ret->prevticks = 0; ret->tid = 0; return ret; } /** * Starts a previously created server using server_new(). Returns * errno if starting a thread failed, after which the caller should * server_free() the given s pointer. */ char server_start(server *s) { return pthread_create(&s->tid, NULL, &server_queuereader, s); } /** * Adds a list of secondary servers to this server. A secondary server * is a server which' queue will be checked when this server has nothing * to do. */ void server_add_secondaries(server *self, server **secondaries, size_t count) { self->secondaries = secondaries; self->secondariescnt = count; } /** * Flags this server as part of a failover cluster, which means the * secondaries are used only to offload on failure, not on queue stress. */ void server_set_failover(server *self) { self->failover = 1; } /** * Sets instance name only used for carbon_ch cluster type. */ void server_set_instance(server *self, char *instance) { if (self->instance != NULL) free(self->instance); self->instance = strdup(instance); } /** * Thin wrapper around the associated queue with the server object. * Returns true if the metric could be queued for sending, or the metric * was dropped because the associated server is down. Returns false * otherwise (when a retry seems like it could succeed shortly). */ inline char server_send(server *s, const char *d, char force) { if (queue_free(s->queue) == 0) { char failure = __sync_add_and_fetch(&(s->failure), 0); if (!force && s->secondariescnt > 0) { size_t i; /* don't immediately drop if we know there are others that * back us up */ for (i = 0; i < s->secondariescnt; i++) { if (!__sync_add_and_fetch(&(s->secondaries[i]->failure), 0)) { failure = 0; break; } } } if (failure || force || s->stallseq == s->maxstalls) { __sync_add_and_fetch(&(s->dropped), 1); /* excess event will be dropped by the enqueue below */ } else { s->stallseq++; __sync_add_and_fetch(&(s->stalls), 1); return 0; } } else { s->stallseq = 0; } queue_enqueue(s->queue, d); return 1; } /** * Tells this server to finish sending pending items from its queue. */ void server_shutdown(server *s) { int i; size_t failures; size_t inqueue; /* this function should only be called on a running server */ if (__sync_bool_compare_and_swap(&(s->keep_running), 0, 0)) return; if (s->secondariescnt > 0) { /* if we have a working connection, or we still have stuff in * our queue, wait for our secondaries, as they might need us, * or we need them */ do { failures = 0; inqueue = 0; for (i = 0; i < s->secondariescnt; i++) { if (__sync_add_and_fetch(&(s->secondaries[i]->failure), 0)) failures++; if (s->secondaries[i]->running) inqueue += queue_len(s->secondaries[i]->queue); } /* loop until we all failed, or nothing is in the queues */ } while (failures != s->secondariescnt && inqueue != 0 && logout("any_of cluster pending %zu metrics " "(with %zu failed nodes)\n", inqueue, failures) >= -1 && usleep((200 + (rand() % 100)) * 1000) <= 0); /* shut down entire cluster */ for (i = 0; i < s->secondariescnt; i++) __sync_bool_compare_and_swap( &(s->secondaries[i]->keep_running), 1, 0); /* to pretend to be dead for above loop (just in case) */ if (inqueue != 0) for (i = 0; i < s->secondariescnt; i++) __sync_add_and_fetch(&(s->secondaries[i]->failure), 1); } __sync_bool_compare_and_swap(&(s->keep_running), 1, 0); } /** * Frees this server and associated resources. This includes joining * the server thread. */ void server_free(server *s) { int err; if (s->tid != 0 && (err = pthread_join(s->tid, NULL)) != 0) logerr("%s:%u: failed to join server thread: %s\n", s->ip, s->port, strerror(err)); s->tid = 0; if (s->ctype == CON_TCP) { size_t qlen = queue_len(s->queue); if (qlen > 0) logerr("dropping %zu metrics for %s:%u\n", qlen, s->ip, s->port); } queue_destroy(s->queue); free(s->batch); if (s->instance) free(s->instance); if (s->saddr != NULL) freeaddrinfo(s->saddr); free((char *)s->ip); s->ip = NULL; free(s); } /** * Swaps the queue between server l to r. This is assumes both l and r * are not running. */ void server_swap_queue(server *l, server *r) { queue *t; assert(l->keep_running == 0 || l->tid == 0); assert(r->keep_running == 0 || r->tid == 0); t = l->queue; l->queue = r->queue; r->queue = t; /* swap associated statistics as well */ l->metrics = r->metrics; l->dropped = r->dropped; l->stalls = r->stalls; l->ticks = r->ticks; l->prevmetrics = r->prevmetrics; l->prevdropped = r->prevdropped; l->prevstalls = r->prevstalls; l->prevticks = r->prevticks; } /** * Returns the ip address this server points to. */ inline const char * server_ip(server *s) { if (s == NULL) return NULL; return s->ip; } /** * Returns the port this server connects at. */ inline unsigned short server_port(server *s) { if (s == NULL) return 0; return s->port; } /** * Returns the instance associated with this server. */ inline char * server_instance(server *s) { return s->instance; } /** * Returns the connection type of this server. */ inline serv_ctype server_ctype(server *s) { if (s == NULL) return CON_PIPE; return s->ctype; } /** * Returns whether the last action on this server caused a failure. */ inline char server_failed(server *s) { if (s == NULL) return 0; return __sync_add_and_fetch(&(s->failure), 0); } /** * Returns the wall-clock time in microseconds (us) consumed sending metrics. */ inline size_t server_get_ticks(server *s) { if (s == NULL) return 0; return __sync_add_and_fetch(&(s->ticks), 0); } /** * Returns the wall-clock time in microseconds (us) consumed since last * call to this function. */ inline size_t server_get_ticks_sub(server *s) { size_t d; if (s == NULL) return 0; d = __sync_add_and_fetch(&(s->ticks), 0) - s->prevticks; s->prevticks += d; return d; } /** * Returns the number of metrics sent since start. */ inline size_t server_get_metrics(server *s) { if (s == NULL) return 0; return __sync_add_and_fetch(&(s->metrics), 0); } /** * Returns the number of metrics sent since last call to this function. */ inline size_t server_get_metrics_sub(server *s) { size_t d; if (s == NULL) return 0; d = __sync_add_and_fetch(&(s->metrics), 0) - s->prevmetrics; s->prevmetrics += d; return d; } /** * Returns the number of metrics dropped since start. */ inline size_t server_get_dropped(server *s) { if (s == NULL) return 0; return __sync_add_and_fetch(&(s->dropped), 0); } /** * Returns the number of metrics dropped since last call to this function. */ inline size_t server_get_dropped_sub(server *s) { size_t d; if (s == NULL) return 0; d = __sync_add_and_fetch(&(s->dropped), 0) - s->prevdropped; s->prevdropped += d; return d; } /** * Returns the number of stalls since start. A stall happens when the * queue is full, but it appears as if it would be a good idea to wait * for a brief period and retry. */ inline size_t server_get_stalls(server *s) { if (s == NULL) return 0; return __sync_add_and_fetch(&(s->stalls), 0); } /** * Returns the number of stalls since last call to this function. */ inline size_t server_get_stalls_sub(server *s) { size_t d; if (s == NULL) return 0; d = __sync_add_and_fetch(&(s->stalls), 0) - s->prevstalls; s->prevstalls += d; return d; } /** * Returns the (approximate) number of metrics waiting to be sent. */ inline size_t server_get_queue_len(server *s) { if (s == NULL) return 0; return queue_len(s->queue); } /** * Returns the allocated size of the queue backing metrics waiting to be * sent. */ inline size_t server_get_queue_size(server *s) { if (s == NULL) return 0; return queue_size(s->queue); } carbon-c-relay-2.5/server.h000066400000000000000000000035621303466152100156440ustar00rootroot00000000000000/* * Copyright 2013-2017 Fabian Groffen * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef SERVER_H #define SERVER_H 1 #include #include "relay.h" #define SERVER_STALL_BITS 4 /* 0 up to 15 */ typedef struct _server server; server *server_new( const char *ip, unsigned short port, serv_ctype ctype, struct addrinfo *saddr, size_t queuesize, size_t batchsize, int maxstalls, unsigned short iotimeout, unsigned int sockbufsize); char server_start(server *s); void server_add_secondaries(server *d, server **sec, size_t cnt); void server_set_failover(server *d); void server_set_instance(server *d, char *inst); char server_send(server *s, const char *d, char force); void server_shutdown(server *s); void server_free(server *s); void server_swap_queue(server *l, server *r); const char *server_ip(server *s); unsigned short server_port(server *s); char *server_instance(server *s); serv_ctype server_ctype(server *s); char server_failed(server *s); size_t server_get_ticks(server *s); size_t server_get_metrics(server *s); size_t server_get_stalls(server *s); size_t server_get_dropped(server *s); size_t server_get_ticks_sub(server *s); size_t server_get_metrics_sub(server *s); size_t server_get_stalls_sub(server *s); size_t server_get_dropped_sub(server *s); size_t server_get_queue_len(server *s); size_t server_get_queue_size(server *s); #endif