1
package client
2
 
3
import (
4
	"context"
5
	"errors"
6
	"path/filepath"
7
	"testing"
8
	"time"
9
 
10
	"go.miragespace.co/specter/spec/protocol"
11
	"go.miragespace.co/specter/spec/rpc"
12
 
13
	"github.com/stretchr/testify/mock"
14
	"github.com/stretchr/testify/require"
15
)
16
 
17
func makeSyncRetryDue(c *Client) {
18
	c.syncStateMu.Lock()
19
	c.nextSync = time.Now().Add(-time.Second)
20
	c.syncStateMu.Unlock()
21
}
22
 
23
func expectStableGateway(c *Client, tunnelClient *mockTunnelClient, maintenanceError error) {
24
	nodes := c.getConnectedNodes()
25
	tunnelClient.TunnelService.On("Ping", mock.Anything, mock.Anything).
26
		Return(&protocol.ClientPingResponse{Node: nodes[0]}, nil)
27
	tunnelClient.TunnelService.On("GetNodes", mock.Anything, mock.Anything).
28
		Return(&protocol.GetNodesResponse{Nodes: nodes}, maintenanceError)
29
}
30
 
31
func TestPublicationRetriesWithStableConnections(t *testing.T) {
32
	c, tunnelClient := newOutcomeTestClient(t, []Tunnel{{Target: "http://localhost:8080"}})
33
	tunnelClient.TunnelService.On("RegisteredHostnames", mock.Anything, mock.Anything).
34
		Return(nil, errors.New("temporary hostname lookup failure")).Once()
35
	tunnelClient.TunnelService.On("RegisteredHostnames", mock.Anything, mock.Anything).
36
		Return(&protocol.RegisteredHostnamesResponse{}, nil).Once()
37
	tunnelClient.TunnelService.On("GenerateHostname", mock.Anything, mock.Anything).
38
		Return(&protocol.GenerateHostnameResponse{Hostname: "generated"}, nil).Once()
39
	tunnelClient.TunnelService.On("PublishTunnel", mock.Anything, mock.Anything).
40
		Return(&protocol.PublishTunnelResponse{Published: c.getConnectedNodes()}, nil).Once()
41
	// Candidate discovery can fail independently of working connected gateways.
42
	expectStableGateway(c, tunnelClient, errors.New("temporary candidate lookup failure"))
43
 
44
	initial := c.SyncConfigTunnels(t.Context())
45
	require.Contains(t, initial.Error, "temporary hostname lookup failure")
46
	require.True(t, c.getStatus().Pending)
47
	require.False(t, initial.Tunnels[0].Published)
48
 
49
	// A normal maintenance cycle respects backoff, even with stable gateways.
50
	c.reconcileConnections(t.Context())
51
	tunnelClient.TunnelService.AssertNumberOfCalls(t, "RegisteredHostnames", 1)
52
	makeSyncRetryDue(c)
53
	c.reconcileConnections(t.Context())
54
 
55
	status := c.getStatus()
56
	require.False(t, status.Pending)
57
	require.Nil(t, status.RetryAt)
58
	require.Empty(t, status.Synchronization.Error)
59
	require.True(t, status.Synchronization.Saved)
60
	require.True(t, status.Synchronization.Tunnels[0].Published)
61
	require.Equal(t, "generated", status.Synchronization.Tunnels[0].Hostname)
62
 
63
	// Converged tunnels do not get republished every maintenance interval.
64
	c.reconcileConnections(t.Context())
65
	tunnelClient.TunnelService.AssertNumberOfCalls(t, "PublishTunnel", 1)
66
}
67
 
68
func TestPublicationRetryKeepsAssignedNamesAndSuccessfulTunnels(t *testing.T) {
69
	c, tunnelClient := newOutcomeTestClient(t, []Tunnel{
70
		{Hostname: "already-published", Target: "http://localhost:8080"},
71
		{Target: "tcp://localhost:5432"},
72
	})
73
	tunnelClient.TunnelService.On("RegisteredHostnames", mock.Anything, mock.Anything).
74
		Return(&protocol.RegisteredHostnamesResponse{}, nil).Once()
75
	tunnelClient.TunnelService.On("GenerateHostname", mock.Anything, mock.Anything).
76
		Return(&protocol.GenerateHostnameResponse{Hostname: "generated"}, nil).Once()
77
	tunnelClient.TunnelService.On("PublishTunnel", mock.Anything, mock.MatchedBy(func(req *protocol.PublishTunnelRequest) bool {
78
		return req.GetHostname() == "already-published"
79
	})).Return(&protocol.PublishTunnelResponse{Published: c.getConnectedNodes()}, nil).Once()
80
	tunnelClient.TunnelService.On("PublishTunnel", mock.Anything, mock.MatchedBy(func(req *protocol.PublishTunnelRequest) bool {
81
		return req.GetHostname() == "generated"
82
	})).Return(nil, errors.New("temporary publication failure")).Once()
83
	tunnelClient.TunnelService.On("PublishTunnel", mock.Anything, mock.MatchedBy(func(req *protocol.PublishTunnelRequest) bool {
84
		return req.GetHostname() == "generated"
85
	})).Return(&protocol.PublishTunnelResponse{Published: c.getConnectedNodes()}, nil).Once()
86
	originalPath := c.Configuration.path
87
	c.Configuration.path = filepath.Join(originalPath, "client.yml")
88
	initial := c.SyncConfigTunnels(t.Context())
89
	require.False(t, initial.Saved)
90
	require.Contains(t, initial.Error, "temporary publication failure")
91
	require.Equal(t, "generated", c.GetCurrentConfig().Tunnels[1].Hostname)
92
 
93
	// Publication may recover without retrying the failed save.
94
	c.Configuration.path = originalPath
95
	c.syncConfigTunnels(t.Context(), false)
96
	status := c.getStatus()
97
	require.False(t, status.Pending)
98
	require.Nil(t, status.RetryAt)
99
	require.False(t, status.Synchronization.Saved)
100
	require.Contains(t, status.Synchronization.Error, "not saved")
101
	for _, tunnel := range status.Synchronization.Tunnels {
102
		require.True(t, tunnel.Published)
103
	}
104
	onDisk, err := NewConfig(originalPath)
105
	require.NoError(t, err)
106
	require.Empty(t, onDisk.Tunnels[1].Hostname, "publication retry must leave the file untouched")
107
	tunnelClient.TunnelService.AssertNumberOfCalls(t, "GenerateHostname", 1)
108
	tunnelClient.TunnelService.AssertNumberOfCalls(t, "PublishTunnel", 3)
109
}
110
 
111
func TestAmbiguousHostnameGenerationReusesRegisteredName(t *testing.T) {
112
	c, tunnelClient := newOutcomeTestClient(t, []Tunnel{{Target: "http://localhost:8080"}})
113
	tunnelClient.TunnelService.On("RegisteredHostnames", mock.Anything, mock.Anything).
114
		Return(&protocol.RegisteredHostnamesResponse{}, nil).Once()
115
	tunnelClient.TunnelService.On("RegisteredHostnames", mock.Anything, mock.Anything).
116
		Return(&protocol.RegisteredHostnamesResponse{Hostnames: []string{"created-before-response-failed"}}, nil).Once()
117
	tunnelClient.TunnelService.On("GenerateHostname", mock.Anything, mock.Anything).
118
		Return(nil, errors.New("response lost after hostname creation")).Once()
119
	tunnelClient.TunnelService.On("PublishTunnel", mock.Anything, mock.MatchedBy(func(req *protocol.PublishTunnelRequest) bool {
120
		return req.GetHostname() == "created-before-response-failed"
121
	})).Return(&protocol.PublishTunnelResponse{Published: c.getConnectedNodes()}, nil).Once()
122
	require.NotEmpty(t, c.SyncConfigTunnels(t.Context()).Error)
123
	c.syncConfigTunnels(t.Context(), false)
124
	require.False(t, c.getStatus().Pending)
125
	tunnelClient.TunnelService.AssertNumberOfCalls(t, "GenerateHostname", 1)
126
}
127
 
128
func TestPublicationRetryStopsOnClientClose(t *testing.T) {
129
	c, tunnelClient := newOutcomeTestClient(t, []Tunnel{{Hostname: "configured", Target: "http://localhost:8080"}})
130
	c.closeCh = make(chan struct{})
131
	tunnelClient.TunnelService.On("RegisteredHostnames", mock.Anything, mock.Anything).
132
		Return(&protocol.RegisteredHostnamesResponse{}, nil).Once()
133
	tunnelClient.TunnelService.On("PublishTunnel", mock.Anything, mock.Anything).
134
		Return(nil, errors.New("initial publication failure")).Once()
135
	entered := make(chan struct{})
136
	tunnelClient.TunnelService.On("PublishTunnel", mock.Anything, mock.Anything).
137
		Run(func(args mock.Arguments) {
138
			close(entered)
139
			<-args.Get(0).(context.Context).Done()
140
		}).Return(nil, context.Canceled).Once()
141
	expectStableGateway(c, tunnelClient, nil)
142
	require.NotEmpty(t, c.SyncConfigTunnels(t.Context()).Error)
143
	makeSyncRetryDue(c)
144
	c.closeWg.Add(1)
145
	finished := make(chan struct{})
146
	go func() {
147
		c.periodicReconnection(t.Context())
148
		close(finished)
149
	}()
150
	select {
151
	case <-entered:
152
	case <-time.After(2 * time.Second):
153
		t.Fatal("background retry never reached publication")
154
	}
155
	close(c.closeCh)
156
	select {
157
	case <-finished:
158
	case <-time.After(time.Second):
159
		t.Fatal("background retry did not stop after client close")
160
	}
161
}
162
 
163
func TestPublicationRetryRepairsPartialGatewayAcknowledgement(t *testing.T) {
164
	c, tunnelClient := newOutcomeTestClient(t, []Tunnel{{Hostname: "configured", Target: "http://localhost:8080"}})
165
	c.connections.Store("second.example.com", &protocol.Node{Id: 2, Address: "second.example.com"})
166
	nodes := c.getConnectedNodes()
167
	tunnelClient.TunnelService.On("RegisteredHostnames", mock.Anything, mock.Anything).
168
		Return(&protocol.RegisteredHostnamesResponse{}, nil).Once()
169
	tunnelClient.TunnelService.On("PublishTunnel", mock.Anything, mock.Anything).
170
		Return(&protocol.PublishTunnelResponse{Published: nodes[:1]}, nil).Once()
171
	tunnelClient.TunnelService.On("PublishTunnel", mock.Anything, mock.Anything).
172
		Return(&protocol.PublishTunnelResponse{Published: nodes}, nil).Once()
173
	initial := c.SyncConfigTunnels(t.Context())
174
	require.Contains(t, initial.Error, "published 1 of 2 connected gateways")
175
	require.True(t, initial.Tunnels[0].Published, "a usable publication remains distinguishable from a total failure")
176
	require.Equal(t, 1, initial.Tunnels[0].PublishedEndpoints)
177
	require.True(t, c.getStatus().Pending)
178
	c.syncConfigTunnels(t.Context(), false)
179
 
180
	status := c.getStatus()
181
	require.False(t, status.Pending)
182
	require.Empty(t, status.Synchronization.Error)
183
	require.Equal(t, 2, status.Synchronization.Tunnels[0].PublishedEndpoints)
184
}
185
 
186
func TestHostnameGenerationFailsOverAfterReconciliation(t *testing.T) {
187
	c, tunnelClient := newOutcomeTestClient(t, []Tunnel{
188
		{Target: "http://localhost:8080"},
189
		{Target: "tcp://localhost:5432"},
190
	})
191
	c.connections.Store("second.example.com", &protocol.Node{Id: 2, Address: "second.example.com"})
192
	nodes := c.getConnectedNodes()
193
	tunnelClient.TunnelService.On("RegisteredHostnames", mock.Anything, mock.Anything).
194
		Return(&protocol.RegisteredHostnamesResponse{}, nil).Twice()
195
	tunnelClient.TunnelService.On("GenerateHostname", mock.MatchedBy(func(ctx context.Context) bool {
196
		return rpc.GetNode(ctx).GetAddress() == nodes[0].GetAddress()
197
	}), mock.Anything).Return(nil, errors.New("preferred gateway cannot create hostnames")).Once()
198
	for _, hostname := range []string{"first-generated", "second-generated"} {
199
		tunnelClient.TunnelService.On("GenerateHostname", mock.MatchedBy(func(ctx context.Context) bool {
200
			return rpc.GetNode(ctx).GetAddress() == nodes[1].GetAddress()
201
		}), mock.Anything).Return(&protocol.GenerateHostnameResponse{Hostname: hostname}, nil).Once()
202
	}
203
	tunnelClient.TunnelService.On("PublishTunnel", mock.Anything, mock.Anything).
204
		Return(&protocol.PublishTunnelResponse{Published: nodes}, nil).Twice()
205
	initial := c.SyncConfigTunnels(t.Context())
206
	require.Contains(t, initial.Error, "preferred gateway cannot create hostnames")
207
	// No other generation can run until registered names reconcile the ambiguous
208
	// response, even when another unnamed tunnel is waiting in this same pass.
209
	tunnelClient.TunnelService.AssertNumberOfCalls(t, "GenerateHostname", 1)
210
	require.Empty(t, c.GetCurrentConfig().Tunnels[0].Hostname)
211
	require.Empty(t, c.GetCurrentConfig().Tunnels[1].Hostname)
212
 
213
	c.syncConfigTunnels(t.Context(), false)
214
	status := c.getStatus()
215
	require.False(t, status.Pending)
216
	require.Empty(t, status.Synchronization.Error)
217
	require.Equal(t, "first-generated", status.Synchronization.Tunnels[0].Hostname)
218
	require.Equal(t, "second-generated", status.Synchronization.Tunnels[1].Hostname)
219
	tunnelClient.TunnelService.AssertNumberOfCalls(t, "RegisteredHostnames", 2)
220
	tunnelClient.TunnelService.AssertNumberOfCalls(t, "GenerateHostname", 3)
221
}