From b8ef91ecd4b99712cd94525d894fa0d72823ef13 Mon Sep 17 00:00:00 2001 From: Yuan Si Date: Mon, 20 Mar 2023 23:23:52 +0800 Subject: [PATCH] Init SifCap Signed-off-by: Yuan Si --- config.yml | 2 + config/yaml.go | 8 + go.mod | 1 + go.sum | 14 ++ main.go | 39 +-- sifcap/sifcap.go | 620 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 667 insertions(+), 17 deletions(-) create mode 100644 sifcap/sifcap.go diff --git a/config.yml b/config.yml index 905bb3e..5ac7c8f 100644 --- a/config.yml +++ b/config.yml @@ -13,3 +13,5 @@ redis: port: "6379" pass: "klsbgames" db: 5 +sifcap: + enabled: false diff --git a/config/yaml.go b/config/yaml.go index 4be358e..ac4691f 100644 --- a/config/yaml.go +++ b/config/yaml.go @@ -18,6 +18,7 @@ type AppConfigs struct { Server ServerConfigs `yaml:"server"` Log LogConfigs `yaml:"log"` Redis RedisConfigs `yaml:"redis"` + SifCap SifCapConfigs `yaml:"sifcap"` } type ServerConfigs struct { @@ -40,6 +41,10 @@ type RedisConfigs struct { Db int `yaml:"db"` } +type SifCapConfigs struct { + Enabled bool `yaml:"enabled"` +} + func DefaultConfigs() *AppConfigs { return &AppConfigs{ AppName: "LL! SIF Private Server", @@ -60,6 +65,9 @@ func DefaultConfigs() *AppConfigs { Pass: "", Db: 0, }, + SifCap: SifCapConfigs{ + Enabled: false, + }, } } diff --git a/go.mod b/go.mod index 6336caf..496a510 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/go-playground/universal-translator v0.18.0 // indirect github.com/go-playground/validator/v10 v10.11.1 // indirect github.com/goccy/go-json v0.9.11 // indirect + github.com/google/gopacket v1.1.19 github.com/json-iterator/go v1.1.12 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/mattn/go-isatty v0.0.16 // indirect diff --git a/go.sum b/go.sum index cef8ff6..0c29be1 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -67,11 +69,20 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -80,11 +91,14 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= diff --git a/main.go b/main.go index 18b7609..efa8e13 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "honoka-chan/config" "honoka-chan/handler" "honoka-chan/middleware" + "honoka-chan/sifcap" "honoka-chan/xclog" "github.com/gin-gonic/gin" @@ -68,24 +69,28 @@ func main() { // } // } - // router - r := gin.Default() - r.Use(middleware.KlabHeader) + if config.Conf.SifCap.Enabled { + sifcap.Start() + } else { + // router + r := gin.Default() + r.Use(middleware.KlabHeader) - r.GET("/webview.php/announce/index", handler.AnnounceIndexHandler) + r.GET("/webview.php/announce/index", handler.AnnounceIndexHandler) - r.POST("/main.php/login/authkey", handler.AuthKeyHandler) - r.POST("/main.php/login/login", handler.LoginHandler) - r.POST("/main.php/user/userInfo", handler.UserInfoHandler) - r.POST("/main.php/gdpr/get", handler.GdprHandler) - r.POST("/main.php/personalnotice/get", handler.PersonalNoticeHandler) - r.POST("/main.php/tos/tosCheck", handler.TosCheckHandler) - r.POST("/main.php/download/event", handler.DownloadEventHandler) - r.POST("/main.php/lbonus/execute", handler.LBonusExecuteHandler) - r.POST("/main.php/api", handler.ApiHandler) - r.POST("/main.php/announce/checkState", handler.AnnounceCheckStateHandler) - r.POST("/main.php/scenario/startup", handler.ScenarioStartupHandler) - r.POST("/main.php/user/setNotificationToken", handler.SetNotificationTokenHandler) + r.POST("/main.php/login/authkey", handler.AuthKeyHandler) + r.POST("/main.php/login/login", handler.LoginHandler) + r.POST("/main.php/user/userInfo", handler.UserInfoHandler) + r.POST("/main.php/gdpr/get", handler.GdprHandler) + r.POST("/main.php/personalnotice/get", handler.PersonalNoticeHandler) + r.POST("/main.php/tos/tosCheck", handler.TosCheckHandler) + r.POST("/main.php/download/event", handler.DownloadEventHandler) + r.POST("/main.php/lbonus/execute", handler.LBonusExecuteHandler) + r.POST("/main.php/api", handler.ApiHandler) + r.POST("/main.php/announce/checkState", handler.AnnounceCheckStateHandler) + r.POST("/main.php/scenario/startup", handler.ScenarioStartupHandler) + r.POST("/main.php/user/setNotificationToken", handler.SetNotificationTokenHandler) - r.Run(":8080") // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080") + r.Run(":8080") // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080") + } } diff --git a/sifcap/sifcap.go b/sifcap/sifcap.go new file mode 100644 index 0000000..c67aaff --- /dev/null +++ b/sifcap/sifcap.go @@ -0,0 +1,620 @@ +// Copyright 2012 Google, Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the LICENSE file in the root of the source +// tree. + +// The pcapdump binary implements a tcpdump-like command line tool with gopacket +// using pcap as a backend data collection mechanism. +package sifcap + +import ( + "bufio" + "bytes" + "compress/gzip" + "encoding/binary" + "encoding/hex" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "os/signal" + "runtime/pprof" + "strings" + "sync" + "time" + + "github.com/google/gopacket" + "github.com/google/gopacket/examples/util" + "github.com/google/gopacket/ip4defrag" + "github.com/google/gopacket/layers" // pulls in all layers decoders + "github.com/google/gopacket/pcap" + "github.com/google/gopacket/reassembly" +) + +var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit") +var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)") +var statsevery = flag.Int("stats", 1000, "Output statistics every N packets") +var lazy = flag.Bool("lazy", false, "If true, do lazy decoding") +var nodefrag = flag.Bool("nodefrag", false, "If true, do not do IPv4 defrag") +var checksum = flag.Bool("checksum", false, "Check TCP checksum") +var nooptcheck = flag.Bool("nooptcheck", false, "Do not check TCP options (useful to ignore MSS on captures with TSO)") +var ignorefsmerr = flag.Bool("ignorefsmerr", false, "Ignore TCP FSM errors") +var allowmissinginit = flag.Bool("allowmissinginit", false, "Support streams without SYN/SYN+ACK/ACK sequence") +var verbose = flag.Bool("verbose", false, "Be verbose") +var debug = flag.Bool("debug", false, "Display debug information") +var quiet = flag.Bool("quiet", true, "Be quiet regarding errors") + +// http +var nohttp = flag.Bool("nohttp", false, "Disable HTTP parsing") + +var hexdump = flag.Bool("dump", false, "Dump HTTP request/response as hex") +var hexdumppkt = flag.Bool("dumppkt", false, "Dump packet as hex") + +// capture +var iface = flag.String("i", "enp8s0", "Interface to read packets from") +var fname = flag.String("r", "", "Filename to read from, overrides -i") +var snaplen = flag.Int("s", 65536, "Snap length (number of bytes max to read per packet") +var tstype = flag.String("timestamp_type", "", "Type of timestamps to use") +var promisc = flag.Bool("promisc", true, "Set promiscuous mode") + +var memprofile = flag.String("memprofile", "", "Write memory profile") + +var stats struct { + ipdefrag int + missedBytes int + pkt int + sz int + totalsz int + rejectFsm int + rejectOpt int + rejectConnFsm int + reassembled int + outOfOrderBytes int + outOfOrderPackets int + biggestChunkBytes int + biggestChunkPackets int + overlapBytes int + overlapPackets int +} + +const closeTimeout time.Duration = time.Hour * 24 // Closing inactive: TODO: from CLI +const timeout time.Duration = time.Minute * 5 // Pending bytes: TODO: from CLI + +/* + * HTTP part + */ + +type httpReader struct { + ident string + isClient bool + bytes chan []byte + data []byte + hexdump bool + parent *tcpStream +} + +func (h *httpReader) Read(p []byte) (int, error) { + ok := true + for ok && len(h.data) == 0 { + h.data, ok = <-h.bytes + } + if !ok || len(h.data) == 0 { + return 0, io.EOF + } + + l := copy(p, h.data) + h.data = h.data[l:] + return l, nil +} + +var outputLevel int +var errorsMap map[string]uint +var errorsMapMutex sync.Mutex +var errors uint + +// Too bad for perf that a... is evaluated +func Error(t string, s string, a ...interface{}) { + errorsMapMutex.Lock() + errors++ + errorsMap[t] = errorsMap[t] + 1 + errorsMapMutex.Unlock() + if outputLevel >= 0 { + fmt.Printf(s, a...) + } +} +func Info(s string, a ...interface{}) { + if outputLevel >= 1 { + fmt.Printf(s, a...) + } +} +func Debug(s string, a ...interface{}) { + if outputLevel >= 2 { + fmt.Printf(s, a...) + } +} + +func (h *httpReader) run(wg *sync.WaitGroup) { + defer wg.Done() + b := bufio.NewReader(h) + for { + if h.isClient { + req, err := http.ReadRequest(b) + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } else if err != nil { + Error("HTTP-request", "HTTP/%s Request error: %s (%v,%+v)\n", h.ident, err, err, err) + continue + } + body, err := io.ReadAll(req.Body) + s := len(body) + if err != nil { + Error("HTTP-request-body", "Got body err: %s\n", err) + } else if h.hexdump { + Info("Body(%d/0x%x)\n%s\n", len(body), len(body), hex.Dump(body)) + } + req.Body.Close() + Info("HTTP/%s Request: %s %s (body:%d)\n", h.ident, req.Method, req.URL, s) + h.parent.Lock() + h.parent.urls = append(h.parent.urls, req.URL.String()) + h.parent.Unlock() + } else { + res, err := http.ReadResponse(b, nil) + var req string + h.parent.Lock() + if len(h.parent.urls) == 0 { + req = "" + } else { + req, h.parent.urls = h.parent.urls[0], h.parent.urls[1:] + } + h.parent.Unlock() + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } else if err != nil { + Error("HTTP-response", "HTTP/%s Response error: %s (%v,%+v)\n", h.ident, err, err, err) + continue + } + body, _ := io.ReadAll(res.Body) + s := len(body) + if err != nil { + Error("HTTP-response-body", "HTTP/%s: failed to get body(parsed len:%d): %s\n", h.ident, s, err) + } + if h.hexdump { + Info("Body(%d/0x%x)\n%s\n", len(body), len(body), hex.Dump(body)) + } + res.Body.Close() + sym := "," + if res.ContentLength > 0 && res.ContentLength != int64(s) { + sym = "!=" + } + contentType, ok := res.Header["Content-Type"] + if !ok { + contentType = []string{http.DetectContentType(body)} + } + encoding := res.Header["Content-Encoding"] + Info("HTTP/%s Response: %s URL:%s (%d%s%d%s) -> %s\n", h.ident, res.Status, req, res.ContentLength, sym, s, contentType, encoding[0]) + if err == nil { + var r io.Reader + r = bytes.NewBuffer(body) + if len(encoding) > 0 && (encoding[0] == "gzip" || encoding[0] == "deflate") { + r, err = gzip.NewReader(r) + if err != nil { + Error("HTTP-gunzip", "Failed to gzip decode: %s", err) + } + } + if err == nil { + res, _ := io.ReadAll(r) + fmt.Println(string(res)) + } + } + } + } +} + +/* + * The TCP factory: returns a new Stream + */ +type tcpStreamFactory struct { + wg sync.WaitGroup + doHTTP bool +} + +func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream { + Debug("* NEW: %s %s\n", net, transport) + fsmOptions := reassembly.TCPSimpleFSMOptions{ + SupportMissingEstablishment: *allowmissinginit, + } + stream := &tcpStream{ + net: net, + transport: transport, + isDNS: tcp.SrcPort == 53 || tcp.DstPort == 53, + isHTTP: (tcp.SrcPort == 80 || tcp.DstPort == 80) && factory.doHTTP, + reversed: tcp.SrcPort == 80, + tcpstate: reassembly.NewTCPSimpleFSM(fsmOptions), + ident: fmt.Sprintf("%s:%s", net, transport), + optchecker: reassembly.NewTCPOptionCheck(), + } + if stream.isHTTP { + stream.client = httpReader{ + bytes: make(chan []byte), + ident: fmt.Sprintf("%s %s", net, transport), + hexdump: *hexdump, + parent: stream, + isClient: true, + } + stream.server = httpReader{ + bytes: make(chan []byte), + ident: fmt.Sprintf("%s %s", net.Reverse(), transport.Reverse()), + hexdump: *hexdump, + parent: stream, + } + factory.wg.Add(2) + go stream.client.run(&factory.wg) + go stream.server.run(&factory.wg) + } + return stream +} + +func (factory *tcpStreamFactory) WaitGoRoutines() { + factory.wg.Wait() +} + +/* + * The assembler context + */ +type Context struct { + CaptureInfo gopacket.CaptureInfo +} + +func (c *Context) GetCaptureInfo() gopacket.CaptureInfo { + return c.CaptureInfo +} + +/* + * TCP stream + */ + +/* It's a connection (bidirectional) */ +type tcpStream struct { + tcpstate *reassembly.TCPSimpleFSM + fsmerr bool + optchecker reassembly.TCPOptionCheck + net, transport gopacket.Flow + isDNS bool + isHTTP bool + reversed bool + client httpReader + server httpReader + urls []string + ident string + sync.Mutex +} + +func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool { + // FSM + if !t.tcpstate.CheckState(tcp, dir) { + Error("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String()) + stats.rejectFsm++ + if !t.fsmerr { + t.fsmerr = true + stats.rejectConnFsm++ + } + if !*ignorefsmerr { + return false + } + } + // Options + err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start) + if err != nil { + Error("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err) + stats.rejectOpt++ + if !*nooptcheck { + return false + } + } + // Checksum + accept := true + if *checksum { + c, err := tcp.ComputeChecksum() + if err != nil { + Error("ChecksumCompute", "%s: Got error computing checksum: %s\n", t.ident, err) + accept = false + } else if c != 0x0 { + Error("Checksum", "%s: Invalid checksum: 0x%x\n", t.ident, c) + accept = false + } + } + if !accept { + stats.rejectOpt++ + } + return accept +} + +func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) { + dir, start, end, skip := sg.Info() + length, saved := sg.Lengths() + // update stats + sgStats := sg.Stats() + if skip > 0 { + stats.missedBytes += skip + } + stats.sz += length - saved + stats.pkt += sgStats.Packets + if sgStats.Chunks > 1 { + stats.reassembled++ + } + stats.outOfOrderPackets += sgStats.QueuedPackets + stats.outOfOrderBytes += sgStats.QueuedBytes + if length > stats.biggestChunkBytes { + stats.biggestChunkBytes = length + } + if sgStats.Packets > stats.biggestChunkPackets { + stats.biggestChunkPackets = sgStats.Packets + } + if sgStats.OverlapBytes != 0 && sgStats.OverlapPackets == 0 { + fmt.Printf("bytes:%d, pkts:%d\n", sgStats.OverlapBytes, sgStats.OverlapPackets) + panic("Invalid overlap") + } + stats.overlapBytes += sgStats.OverlapBytes + stats.overlapPackets += sgStats.OverlapPackets + + var ident string + if dir == reassembly.TCPDirClientToServer { + ident = fmt.Sprintf("%v %v(%s): ", t.net, t.transport, dir) + } else { + ident = fmt.Sprintf("%v %v(%s): ", t.net.Reverse(), t.transport.Reverse(), dir) + } + Debug("%s: SG reassembled packet with %d bytes (start:%v,end:%v,skip:%d,saved:%d,nb:%d,%d,overlap:%d,%d)\n", ident, length, start, end, skip, saved, sgStats.Packets, sgStats.Chunks, sgStats.OverlapBytes, sgStats.OverlapPackets) + if skip == -1 && *allowmissinginit { + // this is allowed + } else if skip != 0 { + // Missing bytes in stream: do not even try to parse it + return + } + data := sg.Fetch(length) + if t.isDNS { + dns := &layers.DNS{} + var decoded []gopacket.LayerType + if len(data) < 2 { + if len(data) > 0 { + sg.KeepFrom(0) + } + return + } + dnsSize := binary.BigEndian.Uint16(data[:2]) + missing := int(dnsSize) - len(data[2:]) + Debug("dnsSize: %d, missing: %d\n", dnsSize, missing) + if missing > 0 { + Info("Missing some bytes: %d\n", missing) + sg.KeepFrom(0) + return + } + p := gopacket.NewDecodingLayerParser(layers.LayerTypeDNS, dns) + err := p.DecodeLayers(data[2:], &decoded) + if err != nil { + Error("DNS-parser", "Failed to decode DNS: %v\n", err) + } else { + Debug("DNS: %s\n", gopacket.LayerDump(dns)) + } + if len(data) > 2+int(dnsSize) { + sg.KeepFrom(2 + int(dnsSize)) + } + } else if t.isHTTP { + if length > 0 { + if *hexdump { + Debug("Feeding http with:\n%s", hex.Dump(data)) + } + if dir == reassembly.TCPDirClientToServer && !t.reversed { + t.client.bytes <- data + } else { + t.server.bytes <- data + } + } + } +} + +func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool { + Debug("%s: Connection closed\n", t.ident) + if t.isHTTP { + close(t.client.bytes) + close(t.server.bytes) + } + // do not remove the connection to allow last ACK + return false +} + +func Start() { + defer util.Run()() + var handle *pcap.Handle + var err error + if *debug { + outputLevel = 2 + } else if *verbose { + outputLevel = 1 + } else if *quiet { + outputLevel = -1 + } + errorsMap = make(map[string]uint) + if *fname != "" { + if handle, err = pcap.OpenOffline(*fname); err != nil { + log.Fatal("PCAP OpenOffline error:", err) + } + } else { + // This is a little complicated because we want to allow all possible options + // for creating the packet capture handle... instead of all this you can + // just call pcap.OpenLive if you want a simple handle. + inactive, err := pcap.NewInactiveHandle(*iface) + if err != nil { + log.Fatalf("could not create: %v", err) + } + defer inactive.CleanUp() + if err = inactive.SetSnapLen(*snaplen); err != nil { + log.Fatalf("could not set snap length: %v", err) + } else if err = inactive.SetPromisc(*promisc); err != nil { + log.Fatalf("could not set promisc mode: %v", err) + } else if err = inactive.SetTimeout(time.Second); err != nil { + log.Fatalf("could not set timeout: %v", err) + } + if *tstype != "" { + if t, err := pcap.TimestampSourceFromString(*tstype); err != nil { + log.Fatalf("Supported timestamp types: %v", inactive.SupportedTimestamps()) + } else if err := inactive.SetTimestampSource(t); err != nil { + log.Fatalf("Supported timestamp types: %v", inactive.SupportedTimestamps()) + } + } + if handle, err = inactive.Activate(); err != nil { + log.Fatal("PCAP Activate error:", err) + } + defer handle.Close() + } + if len(flag.Args()) > 0 { + bpffilter := strings.Join(flag.Args(), " ") + Info("Using BPF filter %q\n", bpffilter) + if err = handle.SetBPFFilter(bpffilter); err != nil { + log.Fatal("BPF filter error:", err) + } + } + + var dec gopacket.Decoder + var ok bool + decoder_name := *decoder + if decoder_name == "" { + decoder_name = handle.LinkType().String() + } + if dec, ok = gopacket.DecodersByLayerName[decoder_name]; !ok { + log.Fatalln("No decoder named", decoder_name) + } + source := gopacket.NewPacketSource(handle, dec) + source.Lazy = *lazy + source.NoCopy = true + Info("Starting to read packets\n") + count := 0 + bytes := int64(0) + start := time.Now() + defragger := ip4defrag.NewIPv4Defragmenter() + + streamFactory := &tcpStreamFactory{doHTTP: !*nohttp} + streamPool := reassembly.NewStreamPool(streamFactory) + assembler := reassembly.NewAssembler(streamPool) + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, os.Interrupt) + + for packet := range source.Packets() { + count++ + Debug("PACKET #%d\n", count) + data := packet.Data() + bytes += int64(len(data)) + if *hexdumppkt { + Debug("Packet content (%d/0x%x)\n%s\n", len(data), len(data), hex.Dump(data)) + } + + // defrag the IPv4 packet if required + if !*nodefrag { + ip4Layer := packet.Layer(layers.LayerTypeIPv4) + if ip4Layer == nil { + continue + } + ip4 := ip4Layer.(*layers.IPv4) + l := ip4.Length + newip4, err := defragger.DefragIPv4(ip4) + if err != nil { + log.Fatalln("Error while de-fragmenting", err) + } else if newip4 == nil { + Debug("Fragment...\n") + continue // packet fragment, we don't have whole packet yet. + } + if newip4.Length != l { + stats.ipdefrag++ + Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType()) + pb, ok := packet.(gopacket.PacketBuilder) + if !ok { + panic("Not a PacketBuilder") + } + nextDecoder := newip4.NextLayerType() + nextDecoder.Decode(newip4.Payload, pb) + } + } + + tcp := packet.Layer(layers.LayerTypeTCP) + if tcp != nil { + tcp := tcp.(*layers.TCP) + if *checksum { + err := tcp.SetNetworkLayerForChecksum(packet.NetworkLayer()) + if err != nil { + log.Fatalf("Failed to set network layer for checksum: %s\n", err) + } + } + c := Context{ + CaptureInfo: packet.Metadata().CaptureInfo, + } + stats.totalsz += len(tcp.Payload) + assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c) + } + if count%*statsevery == 0 { + ref := packet.Metadata().CaptureInfo.Timestamp + flushed, closed := assembler.FlushWithOptions(reassembly.FlushOptions{T: ref.Add(-timeout), TC: ref.Add(-closeTimeout)}) + Debug("Forced flush: %d flushed, %d closed (%s)", flushed, closed, ref) + } + + done := *maxcount > 0 && count >= *maxcount + if count%*statsevery == 0 || done { + errorsMapMutex.Lock() + errorMapLen := len(errorsMap) + errorsMapMutex.Unlock() + fmt.Fprintf(os.Stderr, "Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)\n", count, bytes, time.Since(start), errors, errorMapLen) + } + select { + case <-signalChan: + fmt.Fprintf(os.Stderr, "\nCaught SIGINT: aborting\n") + done = true + default: + // NOP: continue + } + if done { + break + } + } + + closed := assembler.FlushAll() + Debug("Final flush: %d closed", closed) + if outputLevel >= 2 { + streamPool.Dump() + } + + if *memprofile != "" { + f, err := os.Create(*memprofile) + if err != nil { + log.Fatal(err) + } + pprof.WriteHeapProfile(f) + f.Close() + } + + streamFactory.WaitGoRoutines() + Debug("%s\n", assembler.Dump()) + if !*nodefrag { + fmt.Printf("IPdefrag:\t\t%d\n", stats.ipdefrag) + } + fmt.Printf("TCP stats:\n") + fmt.Printf(" missed bytes:\t\t%d\n", stats.missedBytes) + fmt.Printf(" total packets:\t\t%d\n", stats.pkt) + fmt.Printf(" rejected FSM:\t\t%d\n", stats.rejectFsm) + fmt.Printf(" rejected Options:\t%d\n", stats.rejectOpt) + fmt.Printf(" reassembled bytes:\t%d\n", stats.sz) + fmt.Printf(" total TCP bytes:\t%d\n", stats.totalsz) + fmt.Printf(" conn rejected FSM:\t%d\n", stats.rejectConnFsm) + fmt.Printf(" reassembled chunks:\t%d\n", stats.reassembled) + fmt.Printf(" out-of-order packets:\t%d\n", stats.outOfOrderPackets) + fmt.Printf(" out-of-order bytes:\t%d\n", stats.outOfOrderBytes) + fmt.Printf(" biggest-chunk packets:\t%d\n", stats.biggestChunkPackets) + fmt.Printf(" biggest-chunk bytes:\t%d\n", stats.biggestChunkBytes) + fmt.Printf(" overlap packets:\t%d\n", stats.overlapPackets) + fmt.Printf(" overlap bytes:\t\t%d\n", stats.overlapBytes) + fmt.Printf("Errors: %d\n", errors) + for e, _ := range errorsMap { + fmt.Printf(" %s:\t\t%d\n", e, errorsMap[e]) + } +}