get original dst failed in container

117 Views Asked by At

I am designing an http service and need to get the real destination IP. The network on the host looks like this(just like k8s kubeproxy doing):

traffic arrived host -> host ipvs -> docker bridge -> container  

My service(golang) runs in container, but there is an IPVS load balancer that do dnat in front, so I need to get original dst in some way, I know I can use syscall.GetsockoptIPv6Mreq to get real dst, but it just work when service run in host,if run in container,the code failed with err:syscall.GetsockoptIPv6Mreq: no such file or directory

container use docker, run in bridge mod, this is my service code:

package main

import (
   "context"
   "fmt"
   "net"
   "net/http"
   "syscall"
)

const SO_ORIGINAL_DST = 80

type contextKey struct {
   key string
}

var ConnContextKey = &contextKey{"http-conn"}

func SaveConnInContext(ctx context.Context, c net.Conn) context.Context {
   return context.WithValue(ctx, ConnContextKey, c)
}
func GetConn(r *http.Request) net.Conn {
   return r.Context().Value(ConnContextKey).(net.Conn)
}

func getOriginalDst(conn net.Conn) (string, int, error) {
   tc, ok := conn.(*net.TCPConn)
   if !ok {
      return "", 0, fmt.Errorf("redirect proxy only support tcp")
   }

   f, err := tc.File()
   if err != nil {
      return "", 0, fmt.Errorf("get conn file error, err: %s", err)
   }
   defer f.Close()

   addr, err := syscall.GetsockoptIPv6Mreq(int(f.Fd()), syscall.IPPROTO_IP, SO_ORIGINAL_DST)
   if err != nil {
      return "", 0, fmt.Errorf("syscall.GetsockoptIPv6Mreq: %s", err)
   }

   p0 := int(addr.Multiaddr[2])
   p1 := int(addr.Multiaddr[3])

   port := p0*256 + p1

   ips := addr.Multiaddr[4:8]
   ip := fmt.Sprintf("%d.%d.%d.%d", ips[0], ips[1], ips[2], ips[3])

   return ip, port, nil
}

func hello(w http.ResponseWriter, req *http.Request) {
   conn := GetConn(req)

   ip, port, err := getOriginalDst(conn)
   if err != nil {
      fmt.Fprintf(w, fmt.Sprintf("get original dst failed: %s", err))
      return
   }
   msg := fmt.Sprintf("source addr: %s, server addr: %s, original dst: %s:%d \n",
      req.RemoteAddr, req.Context().Value(http.LocalAddrContextKey), ip, port)
   fmt.Fprintf(w, msg)
}

func version(w http.ResponseWriter, req *http.Request) {
   fmt.Fprintf(w, "v4")
}

func main() {
   http.HandleFunc("/", version)
   http.HandleFunc("/test", hello)

   addr := "0.0.0.0:9090"
   srv := http.Server{
      ConnContext: SaveConnInContext,
   }
   //server.ListenAndServe()
   ln, err := net.Listen("tcp4", addr)
   if err != nil {
      panic(err)
   }
   srv.Serve(ln)
}

if service run on the host it works ok:

  1. sudo iptables -t nat -A OUTPUT -p tcp -m tcp --dport 8080 -j REDIRECT --to-ports 9090
  2. curl localhost:8080/test
  3. get expected answer
source addr: 127.0.0.1:21918, server addr: 127.0.0.1:9090, original dst: 127.0.0.1:9090

If the service run in container then the request will fail

➜  ~ curl 172.17.0.2:9090/test
get original dst failed: syscall.GetsockoptIPv6Mreq: no such file or directory

I have tried many ways. At first I thought it was a permission problem. Adding --privileged=true -u=root is still useless.

I expect the code can work and get original dst in container.

0

There are 0 best solutions below