cmux

README

cmux: Connection Mux

cmux is a generic Go library to multiplex connections based on their payload. Using cmux, you can serve gRPC, SSH, HTTPS, HTTP, Go RPC, and pretty much any other protocol on the same TCP listener.

How-To

Simply create your main listener, create a cmux for that listener, and then match connections:

// Create the main listener.
l, err := net.Listen("tcp", ":23456")
if err != nil {
	log.Fatal(err)
}

// Create a cmux.
m := cmux.New(l)

// Match connections in order:
// First grpc, then HTTP, and otherwise Go RPC/TCP.
grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
httpL := m.Match(cmux.HTTP1Fast())
trpcL := m.Match(cmux.Any()) // Any means anything that is not yet matched.

// Create your protocol servers.
grpcS := grpc.NewServer()
grpchello.RegisterGreeterServer(grpcs, &server{})

httpS := &http.Server{
	Handler: &helloHTTP1Handler{},
}

trpcS := rpc.NewServer()
s.Register(&ExampleRPCRcvr{})

// Use the muxed listeners for your servers.
go grpcS.Serve(grpcL)
go httpS.Serve(httpL)
go trpcS.Accept(trpcL)

// Start serving!
m.Serve()

Take a look at other examples in the GoDoc.

Docs

• GoDocs

Performance

There is room for improvment but, since we are only matching the very first bytes of a connection, the performance overheads on long-lived connections (i.e., RPCs and pipelined HTTP streams) is negligible.

TODO(soheil): Add benchmarks.

Limitations

• TLS: net/http uses a type assertion to identify TLS connections; since cmux's lookahead-implementing connection wraps the underlying TLS connection, this type assertion fails. Because of that, you can serve HTTPS using cmux but http.Request.TLS would not be set in your handlers.

• Different Protocols on The Same Connection: cmux matches the connection when it's accepted. For example, one connection can be either gRPC or REST, but not both. That is, we assume that a client connection is either used for gRPC or REST.

Documentation

Overview

Example

package main

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/rpc"

	"google.golang.org/grpc"

	"golang.org/x/net/context"
	"golang.org/x/net/websocket"

	"github.com/soheilhy/cmux"
	grpchello "google.golang.org/grpc/examples/helloworld/helloworld"
)

type exampleHTTPHandler struct{}

func (h *exampleHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "example http response")
}

func serveHTTP(l net.Listener) {
	s := &http.Server{
		Handler: &exampleHTTPHandler{},
	}
	s.Serve(l)
}

func EchoServer(ws *websocket.Conn) {
	io.Copy(ws, ws)
}

func serveWS(l net.Listener) {
	s := &http.Server{
		Handler: websocket.Handler(EchoServer),
	}
	s.Serve(l)
}

type ExampleRPCRcvr struct{}

func (r *ExampleRPCRcvr) Cube(i int, j *int) error {
	*j = i * i
	return nil
}

func serveRPC(l net.Listener) {
	s := rpc.NewServer()
	s.Register(&ExampleRPCRcvr{})
	s.Accept(l)
}

type grpcServer struct{}

func (s *grpcServer) SayHello(ctx context.Context, in *grpchello.HelloRequest) (
	*grpchello.HelloReply, error) {

	return &grpchello.HelloReply{Message: "Hello " + in.Name + " from cmux"}, nil
}

func serveGRPC(l net.Listener) {
	grpcs := grpc.NewServer()
	grpchello.RegisterGreeterServer(grpcs, &grpcServer{})
	grpcs.Serve(l)
}

func main() {
	l, err := net.Listen("tcp", "127.0.0.1:50051")
	if err != nil {
		log.Fatal(err)
	}

	m := cmux.New(l)

	// We first match the connection against HTTP2 fields. If matched, the
	// connection will be sent through the "grpcl" listener.
	grpcl := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
	//Otherwise, we match it againts a websocket upgrade request.
	wsl := m.Match(cmux.HTTP1HeaderField("Upgrade", "websocket"))

	// Otherwise, we match it againts HTTP1 methods. If matched,
	// it is sent through the "httpl" listener.
	httpl := m.Match(cmux.HTTP1Fast())
	// If not matched by HTTP, we assume it is an RPC connection.
	rpcl := m.Match(cmux.Any())

	// Then we used the muxed listeners.
	go serveGRPC(grpcl)
	go serveWS(wsl)
	go serveHTTP(httpl)
	go serveRPC(rpcl)

	m.Serve()
}

Example (BothHTTPAndHTTPS)

This is an example for serving HTTP and HTTPS on the same port.

package main

import (
	"crypto/rand"
	"crypto/tls"
	"fmt"
	"log"
	"net"
	"net/http"

	"github.com/soheilhy/cmux"
)

type anotherHTTPHandler struct{}

func (h *anotherHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "example http response")
}

func serveHTTP1(l net.Listener) {
	s := &http.Server{
		Handler: &anotherHTTPHandler{},
	}
	s.Serve(l)
}

func serveHTTPS(l net.Listener) {
	// Load certificates.
	certificate, err := tls.LoadX509KeyPair("cert.pem", "key.pem")
	if err != nil {
		log.Fatal(err)
	}

	config := &tls.Config{
		Certificates: []tls.Certificate{certificate},
		Rand:         rand.Reader,
	}

	// Create TLS listener.
	tlsl := tls.NewListener(l, config)

	// Serve HTTP over TLS.
	serveHTTP1(tlsl)
}

// This is an example for serving HTTP and HTTPS on the same port.
func main() {
	// Create the TCP listener.
	l, err := net.Listen("tcp", "127.0.0.1:50051")
	if err != nil {
		log.Fatal(err)
	}

	// Create a mux.
	m := cmux.New(l)

	// We first match on HTTP 1.1 methods.
	httpl := m.Match(cmux.HTTP1Fast())

	// If not matched, we assume that its TLS.
	//
	// Note that you can take this listener, do TLS handshake and
	// create another mux to multiplex the connections over TLS.
	tlsl := m.Match(cmux.Any())

	go serveHTTP1(httpl)
	go serveHTTPS(tlsl)

	m.Serve()
}

Example (RecursiveCmux)

This is an example for serving HTTP, HTTPS, and GoRPC/TLS on the same port.

package main

import (
	"crypto/rand"
	"crypto/tls"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/rpc"

	"github.com/soheilhy/cmux"
)

type recursiveHTTPHandler struct{}

func (h *recursiveHTTPHandler) ServeHTTP(w http.ResponseWriter,
	r *http.Request) {

	fmt.Fprintf(w, "example http response")
}

func recursiveServeHTTP(l net.Listener) {
	s := &http.Server{
		Handler: &recursiveHTTPHandler{},
	}
	s.Serve(l)
}

func tlsListener(l net.Listener) net.Listener {
	// Load certificates.
	certificate, err := tls.LoadX509KeyPair("cert.pem", "key.pem")
	if err != nil {
		log.Fatal(err)
	}

	config := &tls.Config{
		Certificates: []tls.Certificate{certificate},
		Rand:         rand.Reader,
	}

	// Create TLS listener.
	tlsl := tls.NewListener(l, config)
	return tlsl
}

type RecursiveRPCRcvr struct{}

func (r *RecursiveRPCRcvr) Cube(i int, j *int) error {
	*j = i * i
	return nil
}

func recursiveServeRPC(l net.Listener) {
	s := rpc.NewServer()
	s.Register(&RecursiveRPCRcvr{})
	s.Accept(l)
}

// This is an example for serving HTTP, HTTPS, and GoRPC/TLS on the same port.
func main() {
	// Create the TCP listener.
	l, err := net.Listen("tcp", "127.0.0.1:50051")
	if err != nil {
		log.Fatal(err)
	}

	// Create a mux.
	tcpm := cmux.New(l)

	// We first match on HTTP 1.1 methods.
	httpl := tcpm.Match(cmux.HTTP1Fast())

	// If not matched, we assume that its TLS.
	tlsl := tcpm.Match(cmux.Any())
	tlsl = tlsListener(tlsl)

	// Now, we build another mux recursively to match HTTPS and GoRPC.
	// You can use the same trick for SSH.
	tlsm := cmux.New(tlsl)
	httpsl := tlsm.Match(cmux.HTTP1Fast())
	gorpcl := tlsm.Match(cmux.Any())

	go recursiveServeHTTP(httpl)
	go recursiveServeHTTP(httpsl)
	go recursiveServeRPC(gorpcl)

	go tlsm.Serve()
	tcpm.Serve()
}

Index

• Variables
• type CMux
  • func New(l net.Listener) CMux
• type ErrNotMatched
  • func (e ErrNotMatched) Error() string
  • func (e ErrNotMatched) Temporary() bool
  • func (e ErrNotMatched) Timeout() bool
• type ErrorHandler
• type Matcher
  • func Any() Matcher
  • func HTTP1() Matcher
  • func HTTP1Fast(extMethods ...string) Matcher
  • func HTTP1HeaderField(name, value string) Matcher
  • func HTTP2() Matcher
  • func HTTP2HeaderField(name, value string) Matcher
  • func PrefixMatcher(strs ...string) Matcher
• type MuxConn
  • func (m *MuxConn) Read(b []byte) (n int, err error)

Examples

• Package
• Package (BothHTTPAndHTTPS)
• Package (RecursiveCmux)

Variables

var (
	ErrListenerClosed = errListenerClosed("mux: listener closed")
)

Types

type CMux

type CMux interface {
	// Match returns a net.Listener that sees (i.e., accepts) only
	// the connections matched by at least one of the matcher.
	//
	// The order used to call Match determines the priority of matchers.
	Match(matchers ...Matcher) net.Listener
	// Serve starts multiplexing the listener. Serve blocks and perhaps
	// should be invoked concurrently within a go routine.
	Serve() error
	// HandleError registers an error handler that handles listener errors.
	HandleError(h ErrorHandler)
}

CMux is a multiplexer for network connections.

func New

func New(l net.Listener) CMux

New instantiates a new connection multiplexer.

type ErrNotMatched

type ErrNotMatched struct {
	// contains filtered or unexported fields
}

ErrNotMatched is returned whenever a connection is not matched by any of the matchers registered in the multiplexer.

func (ErrNotMatched) Error

func (e ErrNotMatched) Error() string

func (ErrNotMatched) Temporary

func (e ErrNotMatched) Temporary() bool

func (ErrNotMatched) Timeout

func (e ErrNotMatched) Timeout() bool

type ErrorHandler

type ErrorHandler func(err error) (ok bool)

ErrorHandler handles an error and returns whether the mux should continue serving the listener.

type Matcher

type Matcher func(r io.Reader) (ok bool)

Matcher matches a connection based on its content.

func Any

func Any() Matcher

Any is a Matcher that matches any connection.

func HTTP1

func HTTP1() Matcher

HTTP1 parses the first line or upto 4096 bytes of the request to see if the conection contains an HTTP request.

func HTTP1Fast

func HTTP1Fast(extMethods ...string) Matcher

HTTP1Fast only matches the methods in the HTTP request.

This matcher is very optimistic: if it returns true, it does not mean that the request is a valid HTTP response. If you want a correct but slower HTTP1 matcher, use HTTP1 instead.

func HTTP1HeaderField

func HTTP1HeaderField(name, value string) Matcher

HTTP1HeaderField returns a matcher matching the header fields of the first request of an HTTP 1 connection.

func HTTP2

func HTTP2() Matcher

HTTP2 parses the frame header of the first frame to detect whether the connection is an HTTP2 connection.

func HTTP2HeaderField

func HTTP2HeaderField(name, value string) Matcher

HTTP2HeaderField resturns a matcher matching the header fields of the first headers frame.

func PrefixMatcher

func PrefixMatcher(strs ...string) Matcher

PrefixMatcher returns a matcher that matches a connection if it starts with any of the strings in strs.

type MuxConn

type MuxConn struct {
	net.Conn
	// contains filtered or unexported fields
}

func (*MuxConn) Read

func (m *MuxConn) Read(b []byte) (n int, err error)