chore(nodes): add a master+node end-to-end harness and CI job
Node sync had no test with two real panels: the single-panel tests either
call the node's controller in-process or hand-set the node-sync scope, so
823db059 shipped a regression no test could see (an admin-token node
dropped every client and enable flag its master pushed).
internal/nodee2e starts a master and a node as separate panel processes
(the DB is a process-wide global, so one process cannot be both) and walks
17 operations per enrollment scope (admin token, node-sync token): adopt,
create/edit on the master, small and bulk attach, client disable, detach,
client delete, inbound disable, traffic pull and reset, node-side delete
mirrored centrally (#6219), master-side delete, node down, node-lost
inbound re-created, create/edit while down, node disabled on the master,
and selected sync mode leaving unselected inbounds alone.
Against the build before 2ffc694a it fails 6 cells for an admin-token node
and 1 for a node-sync one; on main all 34 pass, on Windows and on Linux.
`make node-e2e` builds a stub-dist binary and runs it; ci.yml runs it as
its own job where a SKIP fails the build. Xray is not started, so cells
assert the node's stored state, not traffic through the core.
2026-10-05 04:14:30 -07:00
|
|
|
package nodee2e
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"fmt"
|
2026-10-25 06:13:18 -07:00
|
|
|
"maps"
|
chore(nodes): add a master+node end-to-end harness and CI job
Node sync had no test with two real panels: the single-panel tests either
call the node's controller in-process or hand-set the node-sync scope, so
823db059 shipped a regression no test could see (an admin-token node
dropped every client and enable flag its master pushed).
internal/nodee2e starts a master and a node as separate panel processes
(the DB is a process-wide global, so one process cannot be both) and walks
17 operations per enrollment scope (admin token, node-sync token): adopt,
create/edit on the master, small and bulk attach, client disable, detach,
client delete, inbound disable, traffic pull and reset, node-side delete
mirrored centrally (#6219), master-side delete, node down, node-lost
inbound re-created, create/edit while down, node disabled on the master,
and selected sync mode leaving unselected inbounds alone.
Against the build before 2ffc694a it fails 6 cells for an admin-token node
and 1 for a node-sync one; on main all 34 pass, on Windows and on Linux.
`make node-e2e` builds a stub-dist binary and runs it; ci.yml runs it as
its own job where a SKIP fails the build. Xray is not started, so cells
assert the node's stored state, not traffic through the core.
2026-10-05 04:14:30 -07:00
|
|
|
"net/http"
|
|
|
|
|
"slices"
|
|
|
|
|
"strconv"
|
|
|
|
|
"testing"
|
|
|
|
|
"time"
|
2026-10-25 06:13:18 -07:00
|
|
|
|
|
|
|
|
"github.com/mhsanaei/3x-ui/v3/internal/database/model"
|
chore(nodes): add a master+node end-to-end harness and CI job
Node sync had no test with two real panels: the single-panel tests either
call the node's controller in-process or hand-set the node-sync scope, so
823db059 shipped a regression no test could see (an admin-token node
dropped every client and enable flag its master pushed).
internal/nodee2e starts a master and a node as separate panel processes
(the DB is a process-wide global, so one process cannot be both) and walks
17 operations per enrollment scope (admin token, node-sync token): adopt,
create/edit on the master, small and bulk attach, client disable, detach,
client delete, inbound disable, traffic pull and reset, node-side delete
mirrored centrally (#6219), master-side delete, node down, node-lost
inbound re-created, create/edit while down, node disabled on the master,
and selected sync mode leaving unselected inbounds alone.
Against the build before 2ffc694a it fails 6 cells for an admin-token node
and 1 for a node-sync one; on main all 34 pass, on Windows and on Linux.
`make node-e2e` builds a stub-dist binary and runs it; ci.yml runs it as
its own job where a SKIP fails the build. Xray is not started, so cells
assert the node's stored state, not traffic through the core.
2026-10-05 04:14:30 -07:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// TestNodeSync walks one master/node pair per enrollment scope through every
|
|
|
|
|
// operation that must converge onto the node. Each subtest names its invariant.
|
|
|
|
|
func TestNodeSync(t *testing.T) {
|
|
|
|
|
bin := panelBinary(t)
|
|
|
|
|
for _, scope := range []string{"admin", "node-sync"} {
|
|
|
|
|
t.Run("enrolled with "+scope+" token", func(t *testing.T) {
|
2026-10-05 04:30:18 -07:00
|
|
|
t.Parallel()
|
chore(nodes): add a master+node end-to-end harness and CI job
Node sync had no test with two real panels: the single-panel tests either
call the node's controller in-process or hand-set the node-sync scope, so
823db059 shipped a regression no test could see (an admin-token node
dropped every client and enable flag its master pushed).
internal/nodee2e starts a master and a node as separate panel processes
(the DB is a process-wide global, so one process cannot be both) and walks
17 operations per enrollment scope (admin token, node-sync token): adopt,
create/edit on the master, small and bulk attach, client disable, detach,
client delete, inbound disable, traffic pull and reset, node-side delete
mirrored centrally (#6219), master-side delete, node down, node-lost
inbound re-created, create/edit while down, node disabled on the master,
and selected sync mode leaving unselected inbounds alone.
Against the build before 2ffc694a it fails 6 cells for an admin-token node
and 1 for a node-sync one; on main all 34 pass, on Windows and on Linux.
`make node-e2e` builds a stub-dist binary and runs it; ci.yml runs it as
its own job where a SKIP fails the build. Xray is not started, so cells
assert the node's stored state, not traffic through the core.
2026-10-05 04:14:30 -07:00
|
|
|
runNodeSyncScenarios(t, bin, scope)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type pair struct {
|
|
|
|
|
t *testing.T
|
|
|
|
|
master *panel
|
|
|
|
|
node *panel
|
|
|
|
|
nodeID int
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (pr *pair) nodeInbound(port int) (inboundView, bool) { return pr.node.inboundOnPort(port) }
|
|
|
|
|
|
|
|
|
|
func (pr *pair) masterInbound(port int) (inboundView, bool) {
|
|
|
|
|
for _, ib := range pr.master.inbounds() {
|
|
|
|
|
if ib.Port == port && ib.NodeID != nil && *ib.NodeID == pr.nodeID {
|
|
|
|
|
return ib, true
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return inboundView{}, false
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (pr *pair) waitNode(what string, port int, ok func(inboundView) bool) {
|
|
|
|
|
pr.t.Helper()
|
|
|
|
|
eventually(pr.t, settleTimeout, what, func() (bool, string) {
|
|
|
|
|
ib, found := pr.nodeInbound(port)
|
|
|
|
|
if !found {
|
|
|
|
|
return ok(inboundView{}) && false, fmt.Sprintf("node has no inbound on %d", port)
|
|
|
|
|
}
|
|
|
|
|
return ok(ib), fmt.Sprintf("node inbound %d: enable=%v remark=%q emails=%v", port, ib.Enable, ib.Remark, ib.emails())
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (pr *pair) waitNodeAbsent(what string, port int) {
|
|
|
|
|
pr.t.Helper()
|
|
|
|
|
eventually(pr.t, settleTimeout, what, func() (bool, string) {
|
|
|
|
|
ib, found := pr.nodeInbound(port)
|
|
|
|
|
return !found, fmt.Sprintf("node still has inbound %d with %v", port, ib.emails())
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (pr *pair) bulkAttach(emails []string, masterInboundID int) {
|
|
|
|
|
pr.t.Helper()
|
|
|
|
|
var res struct {
|
|
|
|
|
Attached []string `json:"attached"`
|
|
|
|
|
Errors []string `json:"errors"`
|
|
|
|
|
}
|
|
|
|
|
obj := pr.master.call(http.MethodPost, "/panel/api/clients/bulkAttach", map[string]any{"emails": emails, "inboundIds": []int{masterInboundID}})
|
|
|
|
|
if err := json.Unmarshal(obj, &res); err != nil {
|
|
|
|
|
pr.t.Fatalf("decode bulkAttach: %v", err)
|
|
|
|
|
}
|
|
|
|
|
if len(res.Errors) != 0 || len(res.Attached) != len(emails) {
|
|
|
|
|
pr.t.Fatalf("bulkAttach attached=%d/%d errors=%v", len(res.Attached), len(emails), res.Errors)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func emailRange(prefix string, from, to int) []string {
|
|
|
|
|
out := make([]string, 0, to-from+1)
|
|
|
|
|
for i := from; i <= to; i++ {
|
|
|
|
|
out = append(out, prefix+strconv.Itoa(i))
|
|
|
|
|
}
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func hasAll(have []string, want ...string) bool {
|
|
|
|
|
for _, w := range want {
|
|
|
|
|
if !slices.Contains(have, w) {
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return true
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-25 06:13:18 -07:00
|
|
|
func clientIpSnapshot(t *testing.T, p *panel, email string) (string, map[string]int64) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
var rows []struct {
|
|
|
|
|
ClientEmail string `json:"clientEmail"`
|
|
|
|
|
Ips json.RawMessage `json:"ips"`
|
|
|
|
|
}
|
|
|
|
|
if err := json.Unmarshal(p.call(http.MethodGet, "/panel/api/server/clientIps", nil), &rows); err != nil {
|
|
|
|
|
t.Fatalf("decode %s client IP rows: %v", p.name, err)
|
|
|
|
|
}
|
|
|
|
|
for _, row := range rows {
|
|
|
|
|
if row.ClientEmail != email {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
var entries []model.ClientIpEntry
|
|
|
|
|
if err := json.Unmarshal(row.Ips, &entries); err != nil {
|
|
|
|
|
t.Fatalf("decode %s IP array: %v (%s)", p.name, err, row.Ips)
|
|
|
|
|
}
|
|
|
|
|
got := make(map[string]int64, len(entries))
|
|
|
|
|
for _, entry := range entries {
|
|
|
|
|
if _, duplicate := got[entry.IP]; duplicate {
|
|
|
|
|
t.Fatalf("%s contains duplicate IP %s: %s", p.name, entry.IP, row.Ips)
|
|
|
|
|
}
|
|
|
|
|
got[entry.IP] = entry.Timestamp
|
|
|
|
|
}
|
|
|
|
|
return string(row.Ips), got
|
|
|
|
|
}
|
|
|
|
|
return "", nil
|
|
|
|
|
}
|
|
|
|
|
|
chore(nodes): add a master+node end-to-end harness and CI job
Node sync had no test with two real panels: the single-panel tests either
call the node's controller in-process or hand-set the node-sync scope, so
823db059 shipped a regression no test could see (an admin-token node
dropped every client and enable flag its master pushed).
internal/nodee2e starts a master and a node as separate panel processes
(the DB is a process-wide global, so one process cannot be both) and walks
17 operations per enrollment scope (admin token, node-sync token): adopt,
create/edit on the master, small and bulk attach, client disable, detach,
client delete, inbound disable, traffic pull and reset, node-side delete
mirrored centrally (#6219), master-side delete, node down, node-lost
inbound re-created, create/edit while down, node disabled on the master,
and selected sync mode leaving unselected inbounds alone.
Against the build before 2ffc694a it fails 6 cells for an admin-token node
and 1 for a node-sync one; on main all 34 pass, on Windows and on Linux.
`make node-e2e` builds a stub-dist binary and runs it; ci.yml runs it as
its own job where a SKIP fails the build. Xray is not started, so cells
assert the node's stored state, not traffic through the core.
2026-10-05 04:14:30 -07:00
|
|
|
func runNodeSyncScenarios(t *testing.T, bin, scope string) {
|
|
|
|
|
master := newPanel(t, bin, "master")
|
|
|
|
|
node := newPanel(t, bin, "node")
|
|
|
|
|
linkToken := node.mintToken("master-link", scope)
|
|
|
|
|
master.start()
|
|
|
|
|
node.start()
|
|
|
|
|
pr := &pair{t: t, master: master, node: node}
|
|
|
|
|
|
|
|
|
|
var (
|
|
|
|
|
adoptedPort = freePort(t)
|
|
|
|
|
madePort = freePort(t)
|
|
|
|
|
lostPort = freePort(t)
|
|
|
|
|
droppedPort = freePort(t)
|
|
|
|
|
offlinePort = freePort(t)
|
|
|
|
|
unmanagedPort = freePort(t)
|
|
|
|
|
localPort = freePort(t)
|
|
|
|
|
)
|
|
|
|
|
node.call(http.MethodPost, "/panel/api/inbounds/add", vlessInbound("pre-existing", adoptedPort, nil))
|
|
|
|
|
|
|
|
|
|
local := master.call(http.MethodPost, "/panel/api/inbounds/add", vlessInbound("local-pool", localPort, nil))
|
|
|
|
|
var localIb inboundView
|
|
|
|
|
_ = json.Unmarshal(local, &localIb)
|
|
|
|
|
for _, email := range emailRange("p", 1, 45) {
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/clients/add", map[string]any{
|
|
|
|
|
"client": map[string]any{"email": email, "enable": true}, "inboundIds": []int{localIb.Id},
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var nodeView struct {
|
|
|
|
|
Id int `json:"id"`
|
|
|
|
|
}
|
|
|
|
|
obj := master.call(http.MethodPost, "/panel/api/nodes/add", map[string]any{
|
|
|
|
|
"name": "n1", "scheme": "http", "address": "127.0.0.1", "port": node.port, "basePath": "/",
|
|
|
|
|
"apiToken": linkToken, "enable": true, "allowPrivateAddress": true,
|
|
|
|
|
})
|
|
|
|
|
if err := json.Unmarshal(obj, &nodeView); err != nil || nodeView.Id == 0 {
|
|
|
|
|
t.Fatalf("decode node add: %v (%s)", err, obj)
|
|
|
|
|
}
|
|
|
|
|
pr.nodeID = nodeView.Id
|
|
|
|
|
|
|
|
|
|
var adoptedID, madeID int
|
|
|
|
|
t.Run("an inbound already on the node is adopted by the master", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
eventually(t, settleTimeout, "master adopts the node inbound", func() (bool, string) {
|
|
|
|
|
ib, ok := pr.masterInbound(adoptedPort)
|
|
|
|
|
adoptedID = ib.Id
|
|
|
|
|
return ok, "not adopted yet"
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
if adoptedID == 0 {
|
|
|
|
|
t.Fatal("no adopted inbound; later scenarios depend on it")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
t.Run("an inbound created on the master for the node lands there with its clients", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
obj := master.call(http.MethodPost, "/panel/api/inbounds/add",
|
|
|
|
|
vlessInbound("made-on-master", madePort, &pr.nodeID, vlessClient("m1"), vlessClient("m2")))
|
|
|
|
|
var ib inboundView
|
|
|
|
|
_ = json.Unmarshal(obj, &ib)
|
|
|
|
|
madeID = ib.Id
|
|
|
|
|
pr.waitNode("node holds the master-made inbound", madePort, func(ib inboundView) bool {
|
|
|
|
|
return hasAll(ib.emails(), "m1", "m2")
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("editing the inbound on the master updates the node and keeps its clients", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
body := vlessInbound("renamed-on-master", madePort, &pr.nodeID)
|
|
|
|
|
body["settings"] = `{"decryption":"none"}`
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/inbounds/update/"+strconv.Itoa(madeID), body)
|
|
|
|
|
pr.waitNode("node shows the new remark with both clients", madePort, func(ib inboundView) bool {
|
|
|
|
|
return ib.Remark == "renamed-on-master" && hasAll(ib.emails(), "m1", "m2")
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("attaching a few existing clients reaches the node", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
pr.bulkAttach([]string{"p1", "p2", "p3"}, madeID)
|
|
|
|
|
pr.waitNode("node holds p1..p3", madePort, func(ib inboundView) bool {
|
|
|
|
|
return hasAll(ib.emails(), "m1", "m2", "p1", "p2", "p3")
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("attaching more clients than the per-client push limit reaches the node", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
emails := emailRange("p", 4, 43)
|
|
|
|
|
pr.bulkAttach(emails, adoptedID)
|
|
|
|
|
pr.waitNode("node holds all 40", adoptedPort, func(ib inboundView) bool {
|
|
|
|
|
return hasAll(ib.emails(), emails...)
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("disabling a client on the master disables it on the node", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
mib, _ := pr.masterInbound(madePort)
|
|
|
|
|
entry := mib.client("p1")
|
|
|
|
|
if entry == nil {
|
|
|
|
|
t.Fatalf("master inbound has no p1: %v", mib.emails())
|
|
|
|
|
}
|
|
|
|
|
entry["enable"] = false
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/clients/update/p1", entry)
|
|
|
|
|
pr.waitNode("node p1 disabled", madePort, func(ib inboundView) bool {
|
|
|
|
|
c := ib.client("p1")
|
|
|
|
|
return c != nil && c["enable"] == false
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
2026-11-02 06:27:44 -08:00
|
|
|
t.Run("a stale node disable cannot overwrite a client homed on the master", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
master.stop()
|
|
|
|
|
master.setClientTrafficHome("p3", localIb.Id)
|
|
|
|
|
master.start()
|
|
|
|
|
eventually(t, settleTimeout, "node has no pending master push", func() (bool, string) {
|
|
|
|
|
var current struct {
|
|
|
|
|
ConfigDirty bool `json:"configDirty"`
|
|
|
|
|
}
|
|
|
|
|
if err := json.Unmarshal(master.call(http.MethodGet, "/panel/api/nodes/get/"+strconv.Itoa(pr.nodeID), nil), ¤t); err != nil {
|
|
|
|
|
t.Fatalf("decode node state: %v", err)
|
|
|
|
|
}
|
|
|
|
|
return !current.ConfigDirty, "master still has a pending node push"
|
|
|
|
|
})
|
|
|
|
|
nib, ok := pr.nodeInbound(madePort)
|
|
|
|
|
entry := nib.client("p3")
|
|
|
|
|
if !ok || entry == nil {
|
|
|
|
|
t.Fatal("node has no shared p3 client")
|
|
|
|
|
}
|
|
|
|
|
entry["enable"] = false
|
|
|
|
|
node.call(http.MethodPost, "/panel/api/clients/update/p3", entry)
|
|
|
|
|
readEnabled := func() bool {
|
|
|
|
|
var payload struct {
|
|
|
|
|
Client struct {
|
|
|
|
|
Enable bool `json:"enable"`
|
|
|
|
|
} `json:"client"`
|
|
|
|
|
}
|
|
|
|
|
if err := json.Unmarshal(master.call(http.MethodGet, "/panel/api/clients/get/p3", nil), &payload); err != nil {
|
|
|
|
|
t.Fatalf("decode shared client: %v", err)
|
|
|
|
|
}
|
|
|
|
|
return payload.Client.Enable
|
|
|
|
|
}
|
|
|
|
|
converged := func() bool {
|
|
|
|
|
ib, exists := pr.nodeInbound(madePort)
|
|
|
|
|
client := ib.client("p3")
|
|
|
|
|
return readEnabled() && exists && client != nil && client["enable"] == true
|
|
|
|
|
}
|
|
|
|
|
eventually(t, settleTimeout, "the master lifecycle restores the stale node copy", func() (bool, string) {
|
|
|
|
|
if !readEnabled() {
|
|
|
|
|
t.Fatal("stale node disable overwrote the master's shared client record")
|
|
|
|
|
}
|
|
|
|
|
return converged(), "node copy is still disabled"
|
|
|
|
|
})
|
|
|
|
|
for until := time.Now().Add(12 * time.Second); time.Now().Before(until); {
|
|
|
|
|
if !converged() {
|
|
|
|
|
t.Fatal("shared client enable state drifted across later sync cycles")
|
|
|
|
|
}
|
|
|
|
|
time.Sleep(500 * time.Millisecond)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
chore(nodes): add a master+node end-to-end harness and CI job
Node sync had no test with two real panels: the single-panel tests either
call the node's controller in-process or hand-set the node-sync scope, so
823db059 shipped a regression no test could see (an admin-token node
dropped every client and enable flag its master pushed).
internal/nodee2e starts a master and a node as separate panel processes
(the DB is a process-wide global, so one process cannot be both) and walks
17 operations per enrollment scope (admin token, node-sync token): adopt,
create/edit on the master, small and bulk attach, client disable, detach,
client delete, inbound disable, traffic pull and reset, node-side delete
mirrored centrally (#6219), master-side delete, node down, node-lost
inbound re-created, create/edit while down, node disabled on the master,
and selected sync mode leaving unselected inbounds alone.
Against the build before 2ffc694a it fails 6 cells for an admin-token node
and 1 for a node-sync one; on main all 34 pass, on Windows and on Linux.
`make node-e2e` builds a stub-dist binary and runs it; ci.yml runs it as
its own job where a SKIP fails the build. Xray is not started, so cells
assert the node's stored state, not traffic through the core.
2026-10-05 04:14:30 -07:00
|
|
|
t.Run("detaching a client from the node inbound removes it there", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/clients/p2/detach", map[string]any{"inboundIds": []int{madeID}})
|
|
|
|
|
pr.waitNode("node drops p2", madePort, func(ib inboundView) bool {
|
|
|
|
|
return ib.client("p2") == nil && ib.client("p3") != nil
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("deleting a client on the master removes it from the node", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/clients/del/p4", nil)
|
|
|
|
|
pr.waitNode("node drops p4", adoptedPort, func(ib inboundView) bool {
|
|
|
|
|
return ib.client("p4") == nil && ib.client("p5") != nil
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("switching the inbound off on the master switches it off on the node", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/inbounds/setEnable/"+strconv.Itoa(madeID), map[string]any{"enable": false})
|
|
|
|
|
pr.waitNode("node inbound disabled", madePort, func(ib inboundView) bool { return !ib.Enable })
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("node traffic reaches the master and a master reset clears the node", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
node.call(http.MethodPost, "/panel/api/clients/updateTraffic/p5", map[string]any{"upload": 1000, "download": 2000})
|
|
|
|
|
usage := func(p *panel) int64 {
|
|
|
|
|
var tr struct{ Up, Down int64 }
|
|
|
|
|
_ = json.Unmarshal(p.call(http.MethodGet, "/panel/api/clients/traffic/p5", nil), &tr)
|
|
|
|
|
return tr.Up + tr.Down
|
|
|
|
|
}
|
|
|
|
|
eventually(t, settleTimeout, "master sees p5's node traffic", func() (bool, string) {
|
|
|
|
|
u := usage(master)
|
|
|
|
|
return u == 3000, fmt.Sprintf("master p5 usage %d", u)
|
|
|
|
|
})
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/clients/resetTraffic/p5", nil)
|
|
|
|
|
eventually(t, settleTimeout, "node p5 usage reset", func() (bool, string) {
|
|
|
|
|
u := usage(node)
|
|
|
|
|
return u == 0, fmt.Sprintf("node p5 usage %d", u)
|
|
|
|
|
})
|
|
|
|
|
time.Sleep(12 * time.Second)
|
|
|
|
|
if u := usage(master); u != 0 {
|
|
|
|
|
t.Fatalf("master p5 usage %d after reset settled, want 0", u)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
2026-10-25 06:13:18 -07:00
|
|
|
t.Run("client IP union keeps maximum timestamps stable across sync and an offline node", func(t *testing.T) {
|
|
|
|
|
now := time.Now().Unix()
|
|
|
|
|
post := func(p *panel, ips any) {
|
|
|
|
|
p.call(http.MethodPost, "/panel/api/server/clientIps", []map[string]any{{"clientEmail": "p5", "ips": ips}})
|
|
|
|
|
}
|
|
|
|
|
post(master, []model.ClientIpEntry{
|
|
|
|
|
{IP: "198.51.100.11", Timestamp: now - 30},
|
|
|
|
|
{IP: "198.51.100.12", Timestamp: now - 20},
|
|
|
|
|
})
|
|
|
|
|
post(node, fmt.Sprintf(`[{"ip":"198.51.100.13","timestamp":%d},{"ip":"198.51.100.12","timestamp":%d}]`, now-10, now-5))
|
|
|
|
|
want := map[string]int64{"198.51.100.11": now - 30, "198.51.100.12": now - 5, "198.51.100.13": now - 10}
|
|
|
|
|
waitUnion := func() {
|
|
|
|
|
t.Helper()
|
|
|
|
|
eventually(t, settleTimeout, "both panels retain the exact IP union and maximum timestamps", func() (bool, string) {
|
|
|
|
|
masterRaw, masterIPs := clientIpSnapshot(t, master, "p5")
|
|
|
|
|
nodeRaw, nodeIPs := clientIpSnapshot(t, node, "p5")
|
|
|
|
|
return maps.Equal(masterIPs, want) && maps.Equal(nodeIPs, want) && masterRaw == nodeRaw,
|
|
|
|
|
fmt.Sprintf("master=%s node=%s want=%v", masterRaw, nodeRaw, want)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
waitUnion()
|
|
|
|
|
stableRaw, _ := clientIpSnapshot(t, master, "p5")
|
|
|
|
|
time.Sleep(12 * time.Second)
|
|
|
|
|
for _, p := range []*panel{master, node} {
|
|
|
|
|
raw, got := clientIpSnapshot(t, p, "p5")
|
|
|
|
|
if raw != stableRaw || !maps.Equal(got, want) {
|
|
|
|
|
t.Fatalf("%s IPs drifted after another sync cycle: %s, want %s", p.name, raw, stableRaw)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
node.stop()
|
|
|
|
|
post(master, []model.ClientIpEntry{{IP: "198.51.100.14", Timestamp: now - 1}})
|
|
|
|
|
node.start()
|
|
|
|
|
want["198.51.100.14"] = now - 1
|
|
|
|
|
waitUnion()
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("stopped database observations retain separate GUID attribution buckets", func(t *testing.T) {
|
|
|
|
|
now := time.Now().Unix()
|
|
|
|
|
master.stop()
|
|
|
|
|
node.stop()
|
|
|
|
|
masterIPs := fmt.Sprintf(`[{"ip":"203.0.113.21","timestamp":%d}]`, now-20)
|
|
|
|
|
nodeIPs := fmt.Sprintf(`[{"ip":"203.0.113.22","timestamp":%d}]`, now-10)
|
|
|
|
|
masterGuid := master.seedClientIpAttribution("p5", masterIPs)
|
|
|
|
|
nodeGuid := node.seedClientIpAttribution("p5", nodeIPs)
|
|
|
|
|
if masterGuid == nodeGuid {
|
|
|
|
|
t.Fatalf("independent panels share GUID %q", masterGuid)
|
|
|
|
|
}
|
|
|
|
|
master.start()
|
|
|
|
|
node.start()
|
|
|
|
|
want := map[string]map[string][]model.ClientIpEntry{
|
|
|
|
|
masterGuid: {"p5": {{IP: "203.0.113.21", Timestamp: now - 20}}},
|
|
|
|
|
nodeGuid: {"p5": {{IP: "203.0.113.22", Timestamp: now - 10}}},
|
|
|
|
|
}
|
|
|
|
|
eventually(t, settleTimeout, "master retains the exact independent GUID observation buckets", func() (bool, string) {
|
|
|
|
|
var got map[string]map[string][]model.ClientIpEntry
|
|
|
|
|
raw := master.call(http.MethodPost, "/panel/api/clients/clientIpsByGuid", nil)
|
|
|
|
|
if err := json.Unmarshal(raw, &got); err != nil {
|
|
|
|
|
t.Fatalf("decode attribution: %v", err)
|
|
|
|
|
}
|
|
|
|
|
expected, err := json.Marshal(want)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
actual, err := json.Marshal(got)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return string(actual) == string(expected), fmt.Sprintf("attribution=%s want=%s", actual, expected)
|
|
|
|
|
})
|
|
|
|
|
var nodeTree map[string]map[string][]model.ClientIpEntry
|
|
|
|
|
raw := node.call(http.MethodPost, "/panel/api/clients/clientIpsByGuid", nil)
|
|
|
|
|
if err := json.Unmarshal(raw, &nodeTree); err != nil {
|
|
|
|
|
t.Fatalf("decode node attribution: %v", err)
|
|
|
|
|
}
|
|
|
|
|
if len(nodeTree) != 1 || len(nodeTree[nodeGuid]) != 1 ||
|
|
|
|
|
!slices.Equal(nodeTree[nodeGuid]["p5"], want[nodeGuid]["p5"]) {
|
|
|
|
|
t.Fatalf("node attribution mixed parent observations into its own bucket: %s", raw)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
chore(nodes): add a master+node end-to-end harness and CI job
Node sync had no test with two real panels: the single-panel tests either
call the node's controller in-process or hand-set the node-sync scope, so
823db059 shipped a regression no test could see (an admin-token node
dropped every client and enable flag its master pushed).
internal/nodee2e starts a master and a node as separate panel processes
(the DB is a process-wide global, so one process cannot be both) and walks
17 operations per enrollment scope (admin token, node-sync token): adopt,
create/edit on the master, small and bulk attach, client disable, detach,
client delete, inbound disable, traffic pull and reset, node-side delete
mirrored centrally (#6219), master-side delete, node down, node-lost
inbound re-created, create/edit while down, node disabled on the master,
and selected sync mode leaving unselected inbounds alone.
Against the build before 2ffc694a it fails 6 cells for an admin-token node
and 1 for a node-sync one; on main all 34 pass, on Windows and on Linux.
`make node-e2e` builds a stub-dist binary and runs it; ci.yml runs it as
its own job where a SKIP fails the build. Xray is not started, so cells
assert the node's stored state, not traffic through the core.
2026-10-05 04:14:30 -07:00
|
|
|
t.Run("an inbound deleted on the node is removed from the master too", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/inbounds/add", vlessInbound("deleted-on-node", lostPort, &pr.nodeID, vlessClient("l1")))
|
|
|
|
|
pr.waitNode("node holds the inbound", lostPort, func(ib inboundView) bool { return ib.client("l1") != nil })
|
|
|
|
|
nib, _ := pr.nodeInbound(lostPort)
|
|
|
|
|
node.call(http.MethodPost, "/panel/api/inbounds/del/"+strconv.Itoa(nib.Id), nil)
|
|
|
|
|
eventually(t, settleTimeout, "master mirrors the node-side delete (#6219)", func() (bool, string) {
|
|
|
|
|
_, still := pr.masterInbound(lostPort)
|
|
|
|
|
return !still, "master still has the inbound"
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("deleting the inbound on the master removes it from the node", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
obj := master.call(http.MethodPost, "/panel/api/inbounds/add", vlessInbound("deleted-on-master", droppedPort, &pr.nodeID, vlessClient("d1")))
|
|
|
|
|
var ib inboundView
|
|
|
|
|
_ = json.Unmarshal(obj, &ib)
|
|
|
|
|
pr.waitNode("node holds the inbound", droppedPort, func(ib inboundView) bool { return ib.client("d1") != nil })
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/inbounds/del/"+strconv.Itoa(ib.Id), nil)
|
|
|
|
|
pr.waitNodeAbsent("node drops the inbound", droppedPort)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("a change made while the node is down reaches it once it is back", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
node.stop()
|
|
|
|
|
pr.bulkAttach([]string{"p44"}, adoptedID)
|
|
|
|
|
node.start()
|
|
|
|
|
pr.waitNode("node holds p44 after restart", adoptedPort, func(ib inboundView) bool {
|
|
|
|
|
return ib.client("p44") != nil
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("an inbound the node lost while down is re-created with the master's pending change", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
nib, ok := pr.nodeInbound(adoptedPort)
|
|
|
|
|
if !ok {
|
|
|
|
|
t.Fatal("node has no adopted inbound to lose")
|
|
|
|
|
}
|
|
|
|
|
node.stop()
|
|
|
|
|
node.deleteInboundRow(nib.Id)
|
|
|
|
|
pr.bulkAttach([]string{"p45"}, adoptedID)
|
|
|
|
|
node.start()
|
|
|
|
|
pr.waitNode("node re-creates the inbound with p44 and p45", adoptedPort, func(ib inboundView) bool {
|
|
|
|
|
return ib.client("p44") != nil && ib.client("p45") != nil
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("an inbound created and edited while the node is down lands once it is back", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
node.stop()
|
|
|
|
|
obj := master.call(http.MethodPost, "/panel/api/inbounds/add", vlessInbound("made-while-down", offlinePort, &pr.nodeID, vlessClient("o1")))
|
|
|
|
|
var ib inboundView
|
|
|
|
|
_ = json.Unmarshal(obj, &ib)
|
|
|
|
|
body := vlessInbound("edited-while-down", offlinePort, &pr.nodeID)
|
|
|
|
|
body["settings"] = `{"decryption":"none"}`
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/inbounds/update/"+strconv.Itoa(ib.Id), body)
|
|
|
|
|
node.start()
|
|
|
|
|
pr.waitNode("node holds the edited inbound with o1", offlinePort, func(ib inboundView) bool {
|
|
|
|
|
return ib.Remark == "edited-while-down" && ib.client("o1") != nil
|
|
|
|
|
})
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("a change made while the node is disabled on the master lands once it is re-enabled", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
nodePath := "/panel/api/nodes/setEnable/" + strconv.Itoa(pr.nodeID)
|
|
|
|
|
master.call(http.MethodPost, nodePath, map[string]any{"enable": false})
|
|
|
|
|
pr.bulkAttach([]string{"p6"}, madeID)
|
|
|
|
|
time.Sleep(6 * time.Second)
|
|
|
|
|
if ib, _ := pr.nodeInbound(madePort); ib.client("p6") != nil {
|
|
|
|
|
t.Fatal("a disabled node received a push")
|
|
|
|
|
}
|
|
|
|
|
master.call(http.MethodPost, nodePath, map[string]any{"enable": true})
|
|
|
|
|
pr.waitNode("node holds p6 after re-enable", madePort, func(ib inboundView) bool { return ib.client("p6") != nil })
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("selected sync mode leaves the node's unselected inbounds alone", func(t *testing.T) {
|
|
|
|
|
pr.t = t
|
|
|
|
|
var selected []string
|
|
|
|
|
for _, ib := range master.inbounds() {
|
|
|
|
|
if ib.NodeID != nil && *ib.NodeID == pr.nodeID {
|
|
|
|
|
selected = append(selected, ib.Tag)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/nodes/update/"+strconv.Itoa(pr.nodeID), map[string]any{
|
|
|
|
|
"name": "n1", "scheme": "http", "address": "127.0.0.1", "port": node.port, "basePath": "/",
|
|
|
|
|
"enable": true, "allowPrivateAddress": true, "inboundSyncMode": "selected", "inboundTags": selected,
|
|
|
|
|
})
|
|
|
|
|
node.call(http.MethodPost, "/panel/api/inbounds/add", vlessInbound("node-only", unmanagedPort, nil, vlessClient("u1")))
|
|
|
|
|
pr.bulkAttach([]string{"p7"}, madeID)
|
|
|
|
|
pr.waitNode("selected inbound still converges", madePort, func(ib inboundView) bool { return ib.client("p7") != nil })
|
|
|
|
|
time.Sleep(12 * time.Second)
|
|
|
|
|
if _, adopted := pr.masterInbound(unmanagedPort); adopted {
|
|
|
|
|
t.Fatal("master adopted an unselected node inbound")
|
|
|
|
|
}
|
|
|
|
|
if ib, ok := pr.nodeInbound(unmanagedPort); !ok || ib.client("u1") == nil {
|
|
|
|
|
t.Fatal("reconcile swept or rewrote an unselected node inbound")
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
2026-10-25 00:44:05 -07:00
|
|
|
|
|
|
|
|
func panelTraffic(t *testing.T, p *panel, email string) (int64, int64) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
var tr struct{ Up, Down int64 }
|
|
|
|
|
if err := json.Unmarshal(p.call(http.MethodGet, "/panel/api/clients/traffic/"+email, nil), &tr); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
return tr.Up, tr.Down
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func enrollTrafficNode(t *testing.T, master, node *panel) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
token := node.mintToken("traffic-upstream", "node-sync")
|
|
|
|
|
master.call(http.MethodPost, "/panel/api/nodes/add", map[string]any{
|
|
|
|
|
"name": node.name, "scheme": "http", "address": "127.0.0.1", "port": node.port, "basePath": "/",
|
|
|
|
|
"apiToken": token, "enable": true, "allowPrivateAddress": true,
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestNodeGlobalTrafficHeartbeatChain(t *testing.T) {
|
|
|
|
|
bin := panelBinary(t)
|
|
|
|
|
root := newPanel(t, bin, "traffic-root")
|
|
|
|
|
middle := newPanel(t, bin, "traffic-middle")
|
|
|
|
|
leaf := newPanel(t, bin, "traffic-leaf")
|
|
|
|
|
const email = "chain-usage"
|
|
|
|
|
localIDs := make(map[*panel]int)
|
|
|
|
|
makeClient := func() map[string]any {
|
|
|
|
|
client := vlessClient(email)
|
|
|
|
|
client["subId"] = "chain-sub"
|
|
|
|
|
return client
|
|
|
|
|
}
|
|
|
|
|
for _, p := range []*panel{root, middle, leaf} {
|
|
|
|
|
p.start()
|
|
|
|
|
var ib inboundView
|
|
|
|
|
obj := p.call(http.MethodPost, "/panel/api/inbounds/add", vlessInbound(p.name, freePort(t), nil, makeClient()))
|
|
|
|
|
if err := json.Unmarshal(obj, &ib); err != nil || ib.Id == 0 {
|
|
|
|
|
t.Fatalf("%s local inbound=%s err=%v", p.name, obj, err)
|
|
|
|
|
}
|
|
|
|
|
localIDs[p] = ib.Id
|
|
|
|
|
}
|
|
|
|
|
enrollTrafficNode(t, middle, leaf)
|
|
|
|
|
enrollTrafficNode(t, root, middle)
|
|
|
|
|
eventually(t, 90*time.Second, "chain adopts all zero-traffic baselines", func() (bool, string) {
|
|
|
|
|
rootCount, middleCount := len(root.inbounds()), len(middle.inbounds())
|
|
|
|
|
return rootCount == 3 && middleCount == 2, fmt.Sprintf("root inbounds=%d middle inbounds=%d", rootCount, middleCount)
|
|
|
|
|
})
|
|
|
|
|
waitUsage := func(p *panel, up, down int64) {
|
|
|
|
|
t.Helper()
|
|
|
|
|
eventually(t, 90*time.Second, p.name+" converges global traffic", func() (bool, string) {
|
|
|
|
|
gotUp, gotDown := panelTraffic(t, p, email)
|
|
|
|
|
return gotUp == up && gotDown == down, fmt.Sprintf("%s usage=%d/%d want %d/%d", p.name, gotUp, gotDown, up, down)
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
root.call(http.MethodPost, "/panel/api/clients/updateTraffic/"+email, map[string]any{"upload": 100, "download": 200})
|
|
|
|
|
middle.call(http.MethodPost, "/panel/api/clients/updateTraffic/"+email, map[string]any{"upload": 10, "download": 20})
|
|
|
|
|
leaf.call(http.MethodPost, "/panel/api/clients/updateTraffic/"+email, map[string]any{"upload": 1, "download": 2})
|
|
|
|
|
for _, p := range []*panel{root, middle, leaf} {
|
|
|
|
|
waitUsage(p, 111, 222)
|
|
|
|
|
}
|
|
|
|
|
pushRoot := func(up, down int64) {
|
|
|
|
|
root.call(http.MethodPost, "/panel/api/inbounds/pushClientTraffics", map[string]any{
|
|
|
|
|
"masterGuid": "synthetic-upstream", "traffics": []map[string]any{{"email": email, "up": up, "down": down}},
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
pushRoot(900, 800)
|
|
|
|
|
waitUsage(middle, 900, 800)
|
|
|
|
|
waitUsage(leaf, 900, 800)
|
|
|
|
|
pushRoot(500, 400)
|
|
|
|
|
waitUsage(middle, 500, 400)
|
|
|
|
|
waitUsage(leaf, 500, 400)
|
|
|
|
|
// Each panel independently produced local usage; resetting an overlay cannot erase it.
|
|
|
|
|
// Reset those producers through their real API before testing forwarded zero counters.
|
|
|
|
|
for _, p := range []*panel{leaf, middle, root} {
|
|
|
|
|
p.call(http.MethodPost, "/panel/api/clients/resetTraffic/"+email, nil)
|
|
|
|
|
}
|
|
|
|
|
pushRoot(0, 0)
|
|
|
|
|
for _, p := range []*panel{root, middle, leaf} {
|
|
|
|
|
waitUsage(p, 0, 0)
|
|
|
|
|
}
|
|
|
|
|
leaf.stop()
|
|
|
|
|
pushRoot(77, 88)
|
|
|
|
|
waitUsage(middle, 77, 88)
|
|
|
|
|
leaf.start()
|
|
|
|
|
waitUsage(leaf, 77, 88)
|
|
|
|
|
for _, p := range []*panel{leaf, middle, root} {
|
|
|
|
|
p.call(http.MethodPost, "/panel/api/clients/del/"+email, nil)
|
|
|
|
|
}
|
|
|
|
|
for _, p := range []*panel{root, middle, leaf} {
|
|
|
|
|
eventually(t, 90*time.Second, p.name+" deletes chain client", func() (bool, string) {
|
|
|
|
|
for _, ib := range p.inbounds() {
|
|
|
|
|
if ib.client(email) != nil {
|
|
|
|
|
return false, fmt.Sprintf("%s retains %s", p.name, email)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return true, ""
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
for _, p := range []*panel{leaf, middle, root} {
|
|
|
|
|
p.call(http.MethodPost, "/panel/api/clients/add", map[string]any{
|
|
|
|
|
"client": makeClient(), "inboundIds": []int{localIDs[p]},
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
for _, p := range []*panel{root, middle, leaf} {
|
|
|
|
|
eventually(t, 90*time.Second, p.name+" recreates chain client", func() (bool, string) {
|
|
|
|
|
for _, ib := range p.inbounds() {
|
|
|
|
|
if ib.client(email) != nil {
|
|
|
|
|
return true, ""
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return false, "missing recreated client"
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
pushRoot(9, 8)
|
|
|
|
|
waitUsage(middle, 9, 8)
|
|
|
|
|
waitUsage(leaf, 9, 8)
|
|
|
|
|
}
|