new e2e framework (fatedier#1835)

Makefile

@@ -37,7 +37,10 @@ gotest:
 ci:
 	go test -count=1 -p=1 -v ./tests/...
 
-alltest: gotest ci
+e2e:
+	./hack/run-e2e.sh
+
+alltest: gotest ci e2e
 
 clean:
 	rm -f ./bin/frpc

cmd/frpc/sub/root.go

@@ -212,7 +212,13 @@ func runClient(cfgFilePath string) (err error) {
 	return
 }
 
-func startService(cfg config.ClientCommonConf, pxyCfgs map[string]config.ProxyConf, visitorCfgs map[string]config.VisitorConf, cfgFile string) (err error) {
+func startService(
+	cfg config.ClientCommonConf,
+	pxyCfgs map[string]config.ProxyConf,
+	visitorCfgs map[string]config.VisitorConf,
+	cfgFile string,
+) (err error) {
+
 	log.InitLog(cfg.LogWay, cfg.LogFile, cfg.LogLevel,
 		cfg.LogMaxDays, cfg.DisableLogColor)
 

go.mod

@@ -9,29 +9,31 @@ require (
 	github.com/fatedier/golib v0.0.0-20181107124048-ff8cd814b049
 	github.com/fatedier/kcp-go v2.0.4-0.20190803094908-fe8645b0a904+incompatible
 	github.com/golang/snappy v0.0.0-20170215233205-553a64147049 // indirect
+	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.0
 	github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/klauspost/cpuid v1.2.0 // indirect
 	github.com/klauspost/reedsolomon v1.9.1 // indirect
 	github.com/mattn/go-runewidth v0.0.4 // indirect
+	github.com/onsi/ginkgo v1.12.2
+	github.com/onsi/gomega v1.10.1
 	github.com/pires/go-proxyproto v0.0.0-20190111085350-4d51b51e3bfc
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/prometheus/client_golang v1.4.1
 	github.com/rakyll/statik v0.1.1
 	github.com/rodaine/table v1.0.0
 	github.com/spf13/cobra v0.0.3
-	github.com/spf13/pflag v1.0.1 // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/templexxx/cpufeat v0.0.0-20170927014610-3794dfbfb047 // indirect
 	github.com/templexxx/xor v0.0.0-20170926022130-0af8e873c554 // indirect
 	github.com/tjfoc/gmsm v0.0.0-20171124023159-98aa888b79d8 // indirect
 	github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec
 	github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae // indirect
-	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
+	golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/text v0.3.2 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	gopkg.in/square/go-jose.v2 v2.4.1 // indirect
+	k8s.io/apimachinery v0.18.3
 )

hack/run-e2e.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
+
+which ginkgo &> /dev/null
+if [ $? -ne 0 ]; then
+    echo "ginkgo not found, try to install..."
+    go get -u github.com/onsi/ginkgo/ginkgo
+fi
+
+ginkgo -nodes=1 ${ROOT}/test/e2e -- -frpc-path=${ROOT}/bin/frpc -frps-path=${ROOT}/bin/frps -log-level=debug

test/e2e/e2e.go

@@ -0,0 +1,61 @@
+package e2e
+
+import (
+	"testing"
+
+	"github.com/fatedier/frp/test/e2e/framework"
+	"github.com/fatedier/frp/utils/log"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+	"github.com/onsi/gomega"
+)
+
+var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
+	setupSuite()
+	return nil
+}, func(data []byte) {
+	// Run on all Ginkgo nodes
+	setupSuitePerGinkgoNode()
+})
+
+var _ = ginkgo.SynchronizedAfterSuite(func() {
+	CleanupSuite()
+}, func() {
+	AfterSuiteActions()
+})
+
+// RunE2ETests checks configuration parameters (specified through flags) and then runs
+// E2E tests using the Ginkgo runner.
+// If a "report directory" is specified, one or more JUnit test reports will be
+// generated in this directory, and cluster logs will also be saved.
+// This function is called on each Ginkgo node in parallel mode.
+func RunE2ETests(t *testing.T) {
+	gomega.RegisterFailHandler(framework.Fail)
+
+	log.Info("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
+	ginkgo.RunSpecs(t, "frp e2e suite")
+}
+
+// setupSuite is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step.
+// There are certain operations we only want to run once per overall test invocation
+// (such as deleting old namespaces, or verifying that all system pods are running.
+// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
+// to ensure that these operations only run on the first parallel Ginkgo node.
+//
+// This function takes two parameters: one function which runs on only the first Ginkgo node,
+// returning an opaque byte array, and then a second function which runs on all Ginkgo nodes,
+// accepting the byte array.
+func setupSuite() {
+	// Run only on Ginkgo node 1
+	// TODO
+}
+
+// setupSuitePerGinkgoNode is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step.
+// There are certain operations we only want to run once per overall test invocation on each Ginkgo node
+// such as making some global variables accessible to all parallel executions
+// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
+// Ref: https://onsi.github.io/ginkgo/#parallel-specs
+func setupSuitePerGinkgoNode() {
+	// config.GinkgoConfig.ParallelNode
+}

test/e2e/e2e_test.go

@@ -0,0 +1,34 @@
+package e2e
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/fatedier/frp/test/e2e/framework"
+	"github.com/fatedier/frp/utils/log"
+)
+
+// handleFlags sets up all flags and parses the command line.
+func handleFlags() {
+	framework.RegisterCommonFlags(flag.CommandLine)
+	flag.Parse()
+}
+
+func TestMain(m *testing.M) {
+	// Register test flags, then parse flags.
+	handleFlags()
+
+	if err := framework.ValidateTestContext(&framework.TestContext); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	log.InitLog("console", "", framework.TestContext.LogLevel, 0, true)
+	os.Exit(m.Run())
+}
+
+func TestE2E(t *testing.T) {
+	RunE2ETests(t)
+}

test/e2e/examples.go

@@ -0,0 +1,40 @@
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/fatedier/frp/test/e2e/framework"
+	"github.com/fatedier/frp/test/e2e/framework/consts"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var connTimeout = 5 * time.Second
+
+var _ = Describe("[Feature: Example]", func() {
+	f := framework.NewDefaultFramework()
+
+	Describe("TCP", func() {
+		It("Expose a TCP echo server", func() {
+			serverConf := `
+			[common]
+			bind_port = {{ .PortServer }}
+			`
+
+			clientConf := fmt.Sprintf(`
+			[common]
+			server_port = {{ .PortServer }}
+
+			[tcp]
+			type = tcp
+			local_port = {{ .%s }}
+			remote_port = {{ .PortTCP }}
+			`, framework.TCPEchoServerPort)
+
+			f.RunProcesses([]string{serverConf}, []string{clientConf})
+
+			framework.ExpectTCPReuqest(f.UsedPorts["PortTCP"], []byte(consts.TestString), []byte(consts.TestString), connTimeout)
+		})
+	})
+})

test/e2e/framework/cleanup.go

@@ -0,0 +1,59 @@
+package framework
+
+import (
+	"sync"
+)
+
+// CleanupActionHandle is an integer pointer type for handling cleanup action
+type CleanupActionHandle *int
+type cleanupFuncHandle struct {
+	actionHandle CleanupActionHandle
+	actionHook   func()
+}
+
+var cleanupActionsLock sync.Mutex
+var cleanupHookList = []cleanupFuncHandle{}
+
+// AddCleanupAction installs a function that will be called in the event of the
+// whole test being terminated.  This allows arbitrary pieces of the overall
+// test to hook into SynchronizedAfterSuite().
+// The hooks are called in last-in-first-out order.
+func AddCleanupAction(fn func()) CleanupActionHandle {
+	p := CleanupActionHandle(new(int))
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	c := cleanupFuncHandle{actionHandle: p, actionHook: fn}
+	cleanupHookList = append([]cleanupFuncHandle{c}, cleanupHookList...)
+	return p
+}
+
+// RemoveCleanupAction removes a function that was installed by
+// AddCleanupAction.
+func RemoveCleanupAction(p CleanupActionHandle) {
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	for i, item := range cleanupHookList {
+		if item.actionHandle == p {
+			cleanupHookList = append(cleanupHookList[:i], cleanupHookList[i+1:]...)
+			break
+		}
+	}
+}
+
+// RunCleanupActions runs all functions installed by AddCleanupAction.  It does
+// not remove them (see RemoveCleanupAction) but it does run unlocked, so they
+// may remove themselves.
+func RunCleanupActions() {
+	list := []func(){}
+	func() {
+		cleanupActionsLock.Lock()
+		defer cleanupActionsLock.Unlock()
+		for _, p := range cleanupHookList {
+			list = append(list, p.actionHook)
+		}
+	}()
+	// Run unlocked.
+	for _, fn := range list {
+		fn()
+	}
+}

test/e2e/framework/client.go

@@ -0,0 +1,11 @@
+package framework
+
+type FRPClient struct {
+	port int
+}
+
+func (f *Framework) FRPClient(port int) *FRPClient {
+	return &FRPClient{
+		port: port,
+	}
+}

test/e2e/framework/consts/consts.go

@@ -0,0 +1,5 @@
+package consts
+
+const (
+	TestString = "frp is a fast reverse proxy to help you expose a local server behind a NAT or firewall to the internet."
+)

test/e2e/framework/expect.go

@@ -0,0 +1,55 @@
+package framework
+
+import (
+	"github.com/onsi/gomega"
+)
+
+// ExpectEqual expects the specified two are the same, otherwise an exception raises
+func ExpectEqual(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.Equal(extra), explain...)
+}
+
+// ExpectEqualValues expects the specified two are the same, it not strict about type
+func ExpectEqualValues(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.BeEquivalentTo(extra), explain...)
+}
+
+// ExpectNotEqual expects the specified two are not the same, otherwise an exception raises
+func ExpectNotEqual(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).NotTo(gomega.Equal(extra), explain...)
+}
+
+// ExpectError expects an error happens, otherwise an exception raises
+func ExpectError(err error, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, err).To(gomega.HaveOccurred(), explain...)
+}
+
+// ExpectNoError checks if "err" is set, and if so, fails assertion while logging the error.
+func ExpectNoError(err error, explain ...interface{}) {
+	ExpectNoErrorWithOffset(1, err, explain...)
+}
+
+// ExpectNoErrorWithOffset checks if "err" is set, and if so, fails assertion while logging the error at "offset" levels above its caller
+// (for example, for call chain f -> g -> ExpectNoErrorWithOffset(1, ...) error would be logged for "f").
+func ExpectNoErrorWithOffset(offset int, err error, explain ...interface{}) {
+	gomega.ExpectWithOffset(1+offset, err).NotTo(gomega.HaveOccurred(), explain...)
+}
+
+// ExpectConsistOf expects actual contains precisely the extra elements.  The ordering of the elements does not matter.
+func ExpectConsistOf(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.ConsistOf(extra), explain...)
+}
+
+// ExpectHaveKey expects the actual map has the key in the keyset
+func ExpectHaveKey(actual interface{}, key interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.HaveKey(key), explain...)
+}
+
+// ExpectEmpty expects actual is empty
+func ExpectEmpty(actual interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.BeEmpty(), explain...)
+}
+
+func ExpectTrue(actual interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).Should(gomega.BeTrue(), explain...)
+}