kafkatools

README

Kafka Tools - small cli tools for monitoring and managing Apache Kafka

Installation

$ go get github.com/jurriaan/kafkatools/...

consumer_offsets

CLI tool for fetching kafka consumer group offsets and lag with support for inserting the results into InfluxDB

reset_consumer_group

Simple CLI tool for resetting the offset of a kafka consumer group

License

Copyright 2016 Jurriaan Pruis

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Documentation

Index

• func FetchTopicOffsets(client sarama.Client, offset int64, topic string) (topicOffsets map[int32]TopicPartitionOffset)
• func GenerateOffsetRequests(client sarama.Client, time int64, topics ...string) (requests map[*sarama.Broker]*sarama.OffsetRequest)
• func GetBrokerGroupOffsets(broker *sarama.Broker, groupOffsetChannel chan GroupOffset)
• func GetBrokerTopicOffsets(broker *sarama.Broker, request *sarama.OffsetRequest, ...)
• func GetOffsetFetchRequest(desc *sarama.GroupDescription) *sarama.OffsetFetchRequest
• func GetSaramaClient(brokers ...string) sarama.Client
• func GetSaramaConsumer(brokers string, consumerGroup string, topics []string) *cluster.Consumer
• type GroupOffset
• type GroupOffsetSlice
  • func FetchOffsets(client sarama.Client, offset int64) (groupOffsets GroupOffsetSlice, ...)
  • func (a GroupOffsetSlice) Len() int
  • func (a GroupOffsetSlice) Less(i, j int) bool
  • func (a GroupOffsetSlice) Swap(i, j int)
• type GroupTopicOffset
• type GroupTopicOffsetSlice
  • func (a GroupTopicOffsetSlice) Len() int
  • func (a GroupTopicOffsetSlice) Less(i, j int) bool
  • func (a GroupTopicOffsetSlice) Swap(i, j int)
• type MemberAssignment
  • func ParseMemberAssignment(byteArr []byte) (assignments MemberAssignment)
• type TopicAssignment
• type TopicPartitionOffset
• type TopicPartitionOffsetSlice
  • func (a TopicPartitionOffsetSlice) Len() int
  • func (a TopicPartitionOffsetSlice) Less(i, j int) bool
  • func (a TopicPartitionOffsetSlice) Swap(i, j int)

Functions

func FetchTopicOffsets

func FetchTopicOffsets(client sarama.Client, offset int64, topic string) (topicOffsets map[int32]TopicPartitionOffset)

FetchTopicOffsets fetches topic offsets

func GenerateOffsetRequests

func GenerateOffsetRequests(client sarama.Client, time int64, topics ...string) (requests map[*sarama.Broker]*sarama.OffsetRequest)

GenerateOffsetRequests generates the offset requests which can be used in the GetBrokerTopicOffsets function

func GetBrokerGroupOffsets

func GetBrokerGroupOffsets(broker *sarama.Broker, groupOffsetChannel chan GroupOffset)

GetBrokerGroupOffsets fetches all group offsets for a specific broker

func GetBrokerTopicOffsets

func GetBrokerTopicOffsets(broker *sarama.Broker, request *sarama.OffsetRequest, offsets chan TopicPartitionOffset)

GetBrokerTopicOffsets fetches the offsets for all topics from a specific groker and sends them to the offset topic

func GetOffsetFetchRequest

func GetOffsetFetchRequest(desc *sarama.GroupDescription) *sarama.OffsetFetchRequest

GetOffsetFetchRequest generates a request for the offsets of a specific group

func GetSaramaClient

func GetSaramaClient(brokers ...string) sarama.Client

GetSaramaClient sets up a kafka client

func GetSaramaConsumer

func GetSaramaConsumer(brokers string, consumerGroup string, topics []string) *cluster.Consumer

GetSaramaConsumer returns a high-level kafka consumer

Types

type GroupOffset

type GroupOffset struct {
	Group             string
	GroupTopicOffsets GroupTopicOffsetSlice
}

GroupOffset contains the topic offsets for a specific Group

type GroupOffsetSlice

type GroupOffsetSlice []GroupOffset

GroupOffsetSlice for sorting

func FetchOffsets

func FetchOffsets(client sarama.Client, offset int64) (groupOffsets GroupOffsetSlice, topicOffsets map[string]map[int32]TopicPartitionOffset)

FetchOffsets fetches group and topic offsets (where the topic offset can be sarama.OffsetNewest/OffsetOldest or the time in milliseconds)

func (GroupOffsetSlice) Len

func (a GroupOffsetSlice) Len() int

func (GroupOffsetSlice) Less

func (a GroupOffsetSlice) Less(i, j int) bool

func (GroupOffsetSlice) Swap

func (a GroupOffsetSlice) Swap(i, j int)

type GroupTopicOffset

type GroupTopicOffset struct {
	Topic                 string
	TopicPartitionOffsets TopicPartitionOffsetSlice
}

GroupTopicOffset contains the partition offset of a topic

type GroupTopicOffsetSlice

type GroupTopicOffsetSlice []GroupTopicOffset

GroupTopicOffsetSlice for sorting

func (GroupTopicOffsetSlice) Len

func (a GroupTopicOffsetSlice) Len() int

func (GroupTopicOffsetSlice) Less

func (a GroupTopicOffsetSlice) Less(i, j int) bool

func (GroupTopicOffsetSlice) Swap

func (a GroupTopicOffsetSlice) Swap(i, j int)

type MemberAssignment

type MemberAssignment struct {
	Version     int
	Assignments []TopicAssignment
}

MemberAssignment contains the assignments of a consumer group member

func ParseMemberAssignment

func ParseMemberAssignment(byteArr []byte) (assignments MemberAssignment)

ParseMemberAssignment parses a binary byteArr

type TopicAssignment

type TopicAssignment struct {
	Topic      string
	Partitions []int32
}

TopicAssignment contains the assigned partitions of a topic

type TopicPartitionOffset

type TopicPartitionOffset struct {
	Topic     string
	Partition int32
	Offset    int64
}

TopicPartitionOffset information

type TopicPartitionOffsetSlice

type TopicPartitionOffsetSlice []TopicPartitionOffset

TopicPartitionOffsetSlice for sorting

func (TopicPartitionOffsetSlice) Len

func (a TopicPartitionOffsetSlice) Len() int

func (TopicPartitionOffsetSlice) Less

func (a TopicPartitionOffsetSlice) Less(i, j int) bool

func (TopicPartitionOffsetSlice) Swap

func (a TopicPartitionOffsetSlice) Swap(i, j int)