1 package growthbook
2
3 import (
4 "fmt"
5 "reflect"
6 "regexp"
7 "testing"
8 )
9
10 type trackCall struct {
11 experiment *Experiment
12 result *Result
13 }
14
15 type tracker struct {
16 calls []trackCall
17 cb func(experiment *Experiment, result *Result)
18 }
19
20 func track() *tracker {
21 t := tracker{[]trackCall{}, nil}
22 t.cb = func(experiment *Experiment, result *Result) {
23 t.calls = append(t.calls, trackCall{experiment, result})
24 }
25 return &t
26 }
27
28 func TestExperimentTracking(t *testing.T) {
29 context := NewContext().
30 WithUserAttributes(Attributes{"id": "1"})
31
32 tr := track()
33 gb := New(context).WithTrackingCallback(tr.cb)
34
35 exp1 := NewExperiment("my-tracked-test").WithVariations(0, 1)
36 exp2 := NewExperiment("my-other-tracked-test").WithVariations(0, 1)
37
38 res1 := gb.Run(exp1)
39 gb.Run(exp1)
40 gb.Run(exp1)
41 res4 := gb.Run(exp2)
42 context = context.WithUserAttributes(Attributes{"id": "2"})
43 res5 := gb.Run(exp2)
44
45 if len(tr.calls) != 3 {
46 t.Errorf("expected 3 calls to tracking callback, got %d", len(tr.calls))
47 } else {
48 if !reflect.DeepEqual(tr.calls[0], trackCall{exp1, res1}) {
49 t.Errorf("unexpected callback result")
50 }
51 if !reflect.DeepEqual(tr.calls[1], trackCall{exp2, res4}) {
52 t.Errorf("unexpected callback result")
53 }
54 if !reflect.DeepEqual(tr.calls[2], trackCall{exp2, res5}) {
55 t.Errorf("unexpected callback result")
56 }
57 }
58 }
59
60 func TestExperimentForcesVariationFromOverrides(t *testing.T) {
61 forceVal := 1
62 context := NewContext().
63 WithOverrides(ExperimentOverrides{
64 "forced-test": &ExperimentOverride{
65 Force: &forceVal,
66 }})
67 gb := New(context).
68 WithAttributes(Attributes{"id": "6"})
69
70 res := gb.Run(NewExperiment("forced-test").WithVariations(0, 1))
71
72 if res.VariationID != 1 {
73 t.Error("expected variation ID 1, got", res.VariationID)
74 }
75 if res.InExperiment != true {
76 t.Error("expected InExperiment to be true")
77 }
78 if res.HashUsed != false {
79 t.Error("expected HashUsed to be false")
80 }
81 }
82
83 func TestExperimentCoverageFromOverrides(t *testing.T) {
84 overrideVal := 0.01
85 context := NewContext().
86 WithOverrides(ExperimentOverrides{
87 "my-test": &ExperimentOverride{
88 Coverage: &overrideVal,
89 }})
90 gb := New(context).
91 WithAttributes(Attributes{"id": "1"})
92
93 res := gb.Run(NewExperiment("my-test").WithVariations(0, 1))
94
95 if res.VariationID != 0 {
96 t.Error("expected variation ID 0, got", res.VariationID)
97 }
98 if res.InExperiment != false {
99 t.Error("expected InExperiment to be false")
100 }
101 }
102
103 func TestExperimentDoesNotTrackWhenForcedWithOverrides(t *testing.T) {
104 context := NewContext().
105 WithUserAttributes(Attributes{"id": "6"})
106 tr := track()
107 gb := New(context).WithTrackingCallback(tr.cb)
108 exp := NewExperiment("forced-test").WithVariations(0, 1)
109
110 forceVal := 1
111 context = context.WithOverrides(ExperimentOverrides{
112 "forced-test": &ExperimentOverride{Force: &forceVal},
113 })
114
115 gb.Run(exp)
116
117 if len(tr.calls) != 0 {
118 t.Error("expected 0 calls to tracking callback, got ", len(tr.calls))
119 }
120 }
121
122 func TestExperimentURLFromOverrides(t *testing.T) {
123 urlRe := regexp.MustCompile(`^\/path`)
124 context := NewContext().
125 WithUserAttributes(Attributes{"id": "1"}).
126 WithOverrides(ExperimentOverrides{
127 "my-test": &ExperimentOverride{URL: urlRe},
128 })
129 gb := New(context)
130
131 if gb.Run(NewExperiment("my-test").WithVariations(0, 1)).InExperiment != false {
132 t.Error("expected InExperiment to be false")
133 }
134 }
135
136 func TestExperimentFiltersUserGroups(t *testing.T) {
137 context := NewContext().
138 WithUserAttributes(Attributes{"id": "123"}).
139 WithGroups(map[string]bool{
140 "alpha": true,
141 "beta": true,
142 "internal": false,
143 "qa": false,
144 })
145 gb := New(context)
146
147 exp := NewExperiment("my-test").
148 WithVariations(0, 1).
149 WithGroups("internal", "qa")
150 if gb.Run(exp).InExperiment != false {
151 t.Error("1: expected InExperiment to be false")
152 }
153
154 exp = NewExperiment("my-test").
155 WithVariations(0, 1).
156 WithGroups("internal", "qa", "beta")
157 if gb.Run(exp).InExperiment != true {
158 t.Error("2: expected InExperiment to be true")
159 }
160
161 exp = NewExperiment("my-test").
162 WithVariations(0, 1)
163 if gb.Run(exp).InExperiment != true {
164 t.Error("3: expected InExperiment to be true")
165 }
166 }
167
168 func TestExperimentSetsAttributes(t *testing.T) {
169 attributes := Attributes{
170 "id": "1",
171 "browser": "firefox",
172 }
173 gb := New(nil).WithAttributes(attributes)
174
175 if !reflect.DeepEqual(gb.Attributes(), attributes) {
176 t.Error("expected attributes to match")
177 }
178 }
179
180 func TestExperimentCustomIncludeCallback(t *testing.T) {
181 context := NewContext().
182 WithUserAttributes(Attributes{"id": "1"})
183 gb := New(context)
184
185 exp := NewExperiment("my-test").
186 WithVariations(0, 1).
187 WithIncludeFunction(func() bool { return false })
188
189 if gb.Run(exp).InExperiment != false {
190 t.Error("expected InExperiment to be false")
191 }
192 }
193
194 func TestExperimentTrackingSkippedWhenContextDisabled(t *testing.T) {
195 context := NewContext().
196 WithUserAttributes(Attributes{"id": "1"}).
197 WithEnabled(false)
198 tr := track()
199 gb := New(context).WithTrackingCallback(tr.cb)
200
201 gb.Run(NewExperiment("disabled-test").WithVariations(0, 1))
202
203 if len(tr.calls) != 0 {
204 t.Errorf("expected 0 calls to tracking callback, got %d", len(tr.calls))
205 }
206 }
207
208 func TestExperimentQuerystringForceDisablsTracking(t *testing.T) {
209 context := NewContext().
210 WithUserAttributes(Attributes{"id": "1"}).
211 WithURL(mustParseUrl("http://example.com?forced-test-qs=1"))
212 tr := track()
213 gb := New(context).WithTrackingCallback(tr.cb)
214
215 gb.Run(NewExperiment("forced-test-qs").WithVariations(0, 1))
216
217 if len(tr.calls) != 0 {
218 t.Errorf("expected 0 calls to tracking callback, got %d", len(tr.calls))
219 }
220 }
221
222 func TestExperimentURLTargeting(t *testing.T) {
223 context := NewContext().
224 WithUserAttributes(Attributes{"id": "1"}).
225 WithURL(mustParseUrl("http://example.com"))
226 gb := New(context)
227
228 exp := NewExperiment("my-test").
229 WithVariations(0, 1).
230 WithURL(regexp.MustCompile("^/post/[0-9]+"))
231
232 check := func(icase int, e *Experiment, inExperiment bool, value interface{}) {
233 result := gb.Run(e)
234 if result.InExperiment != inExperiment {
235 t.Errorf("%d: expected InExperiment = %v, got %v",
236 icase, inExperiment, result.InExperiment)
237 }
238 if !reflect.DeepEqual(result.Value, value) {
239 t.Errorf("%d: expected value = %v, got %v",
240 icase, value, result.Value)
241 }
242 }
243
244 check(1, exp, false, 0)
245
246 context = context.WithURL(mustParseUrl("http://example.com/post/123"))
247 check(2, exp, true, 1)
248
249 exp.URL = regexp.MustCompile("http://example.com/post/[0-9]+")
250 check(3, exp, true, 1)
251 }
252
253 func TestExperimentIgnoresDraftExperiments(t *testing.T) {
254 context := NewContext().
255 WithUserAttributes(Attributes{"id": "1"})
256 gb := New(context)
257
258 exp := NewExperiment("my-test").
259 WithStatus(DraftStatus).
260 WithVariations(0, 1)
261
262 res1 := gb.Run(exp)
263 context = context.WithURL(mustParseUrl("http://example.com/?my-test=1"))
264 res2 := gb.Run(exp)
265
266 if res1.InExperiment != false {
267 t.Error("1: expected InExperiment to be false")
268 }
269 if res1.HashUsed != false {
270 t.Error("1: expected HashUsed to be false")
271 }
272 if res1.Value != 0 {
273 t.Errorf("1: expected Value to be 0, got %v", res1.Value)
274 }
275
276 if res2.InExperiment != true {
277 t.Error("2: expected InExperiment to be true")
278 }
279 if res2.HashUsed != false {
280 t.Error("2: expected HashUsed to be false")
281 }
282 if res2.Value != 1 {
283 t.Errorf("2: expected Value to be 1, got %v", res2.Value)
284 }
285 }
286
287 func TestExperimentIgnoresStoppedExperimentsUnlessForced(t *testing.T) {
288 context := NewContext().
289 WithUserAttributes(Attributes{"id": "1"})
290 gb := New(context)
291
292 expLose := NewExperiment("my-test").
293 WithStatus(StoppedStatus).
294 WithVariations(0, 1, 2)
295 expWin := NewExperiment("my-test").
296 WithStatus(StoppedStatus).
297 WithVariations(0, 1, 2).
298 WithForce(2)
299
300 res1 := gb.Run(expLose)
301 res2 := gb.Run(expWin)
302
303 if res1.InExperiment != false {
304 t.Error("1: expected InExperiment to be false")
305 }
306 if res1.HashUsed != false {
307 t.Error("1: expected HashUsed to be false")
308 }
309 if res1.Value != 0 {
310 t.Errorf("1: expected Value to be 0, got %v", res1.Value)
311 }
312
313 if res2.InExperiment != true {
314 t.Error("2: expected InExperiment to be true")
315 }
316 if res2.HashUsed != false {
317 t.Error("2: expected HashUsed to be false")
318 }
319 if res2.Value != 2 {
320 t.Errorf("2: expected Value to be 2, got %v", res2.Value)
321 }
322 }
323
324 func TestExperimentDoesEvenWeighting(t *testing.T) {
325 context := NewContext()
326 gb := New(context)
327
328
329 exp := NewExperiment("my-test").WithVariations(0, 1)
330 variations := map[string]int{
331 "0": 0,
332 "1": 0,
333 "-1": 0,
334 }
335 countVariations(t, context, gb, exp, 1000, variations)
336 if variations["0"] != 503 {
337 t.Errorf("1: expected variations[\"0\"] to be 503, got %v", variations["0"])
338 }
339
340
341 exp = exp.WithCoverage(0.4)
342 variations = map[string]int{
343 "0": 0,
344 "1": 0,
345 "-1": 0,
346 }
347 countVariations(t, context, gb, exp, 10000, variations)
348 if variations["0"] != 2044 {
349 t.Errorf("2: expected variations[\"0\"] to be 2044, got %v", variations["0"])
350 }
351 if variations["1"] != 1980 {
352 t.Errorf("2: expected variations[\"1\"] to be 1980, got %v", variations["0"])
353 }
354 if variations["-1"] != 5976 {
355 t.Errorf("2: expected variations[\"0\"] to be 5976, got %v", variations["0"])
356 }
357
358
359 exp = exp.WithCoverage(0.6).WithVariations(0, 1, 2)
360 variations = map[string]int{
361 "0": 0,
362 "1": 0,
363 "2": 0,
364 "-1": 0,
365 }
366 countVariations(t, context, gb, exp, 10000, variations)
367 expected := map[string]int{
368 "-1": 3913,
369 "0": 2044,
370 "1": 2000,
371 "2": 2043,
372 }
373 if !reflect.DeepEqual(variations, expected) {
374 t.Errorf("3: expected variations counts %#v, git %#v", expected, variations)
375 }
376 }
377
378 func TestExperimentForcesMultipleVariationsAtOnce(t *testing.T) {
379 context := NewContext().
380 WithUserAttributes(Attributes{"id": "1"})
381 gb := New(context)
382
383 exp := NewExperiment("my-test").
384 WithVariations(0, 1)
385
386 res1 := gb.Run(exp)
387 commonCheck(t, 1, res1, true, true, 1)
388
389 gb = gb.WithForcedVariations(ForcedVariationsMap{
390 "my-test": 0,
391 })
392 res2 := gb.Run(exp)
393 commonCheck(t, 2, res2, true, false, 0)
394
395 gb = gb.WithForcedVariations(nil)
396 res3 := gb.Run(exp)
397 commonCheck(t, 3, res3, true, true, 1)
398 }
399
400 func TestExperimentOnceForcesAllVariationsInQAMode(t *testing.T) {
401 context := NewContext().
402 WithUserAttributes(Attributes{"id": "1"}).
403 WithQAMode(true)
404 gb := New(context)
405
406 exp := NewExperiment("my-test").
407 WithVariations(0, 1)
408
409 res1 := gb.Run(exp)
410 commonCheck(t, 1, res1, false, false, 0)
411
412
413 context = context.WithForcedVariations(ForcedVariationsMap{"my-test": 1})
414 res2 := gb.Run(exp)
415 commonCheck(t, 2, res2, true, false, 1)
416
417
418 exp2 := NewExperiment("my-test-2").WithVariations(0, 1).WithForce(1)
419 res3 := gb.Run(exp2)
420 commonCheck(t, 3, res3, true, false, 1)
421 }
422
423 func TestExperimentFiresSubscriptionsCorrectly(t *testing.T) {
424 context := NewContext().
425 WithUserAttributes(Attributes{"id": "1"})
426 gb := New(context)
427
428 fired := false
429 checkFired := func(icase int, f bool) {
430 if fired != f {
431 t.Errorf("%d: expected fired to be %v", icase, f)
432 }
433 }
434
435 unsubscriber := gb.Subscribe(func(experiment *Experiment, result *Result) {
436 fired = true
437 })
438 checkFired(1, false)
439
440 exp := NewExperiment("my-test").WithVariations(0, 1)
441
442
443 gb.Run(exp)
444 checkFired(2, true)
445
446
447 fired = false
448 gb.Run(exp)
449 checkFired(3, false)
450
451
452 unsubscriber()
453 exp2 := NewExperiment("other-test").WithVariations(0, 1)
454 gb.Run(exp2)
455 checkFired(4, false)
456 }
457
458 func TestExperimentStoresAssignedVariations(t *testing.T) {
459 context := NewContext().
460 WithUserAttributes(Attributes{"id": "1"})
461 gb := New(context)
462 gb.Run(NewExperiment("my-test").WithVariations(0, 1))
463 gb.Run(NewExperiment("my-test-3").WithVariations(0, 1))
464
465 assignedVars := gb.GetAllResults()
466
467 if len(assignedVars) != 2 {
468 t.Errorf("expected len(assignedVars) to be 2, got %d", len(assignedVars))
469 }
470 if assignedVars["my-test"].Result.VariationID != 1 {
471 t.Errorf("expected assignedVars[\"my-test\"] to be 1, got %d",
472 assignedVars["my-test"].Result.VariationID)
473 }
474 if assignedVars["my-test-3"].Result.VariationID != 0 {
475 t.Errorf("expected assignedVars[\"my-test-3\"] to be 0, got %d",
476 assignedVars["my-test-3"].Result.VariationID)
477 }
478 }
479
480 func TestExperimentDoesNotHaveBiasWhenUsingNamespaces(t *testing.T) {
481 context := NewContext().
482 WithUserAttributes(Attributes{"id": "1"})
483 gb := New(context)
484
485 variations := map[string]int{
486 "0": 0,
487 "1": 0,
488 "-1": 0,
489 }
490
491 exp := NewExperiment("my-test").
492 WithVariations(0, 1).
493 WithNamespace(&Namespace{"namespace", 0.0, 0.5})
494 countVariations(t, context, gb, exp, 10000, variations)
495
496 expected := map[string]int{
497 "-1": 4973,
498 "0": 2538,
499 "1": 2489,
500 }
501 if !reflect.DeepEqual(variations, expected) {
502 t.Errorf("expected variations counts %#v, git %#v", expected, variations)
503 }
504 }
505
506 func commonCheck(t *testing.T, icase int, res *Result,
507 inExperiment bool, hashUsed bool, value FeatureValue) {
508 if res.InExperiment != inExperiment {
509 t.Errorf("%d: expected InExperiment to be %v", icase, inExperiment)
510 }
511 if res.HashUsed != hashUsed {
512 t.Errorf("%d: expected HashUsed to be %v", icase, hashUsed)
513 }
514 if res.Value != value {
515 t.Errorf("3: expected Value to be %#v, got %#v", value, res.Value)
516 }
517 }
518
519 func countVariations(t *testing.T, context *Context, gb *GrowthBook,
520 exp *Experiment, runs int, variations map[string]int) {
521 for i := 0; i < runs; i++ {
522 context = context.WithUserAttributes(Attributes{"id": fmt.Sprint(i)})
523 res := gb.Run(exp)
524 v := -1
525 ok := false
526 if res.InExperiment {
527 v, ok = res.Value.(int)
528 if !ok {
529 t.Error("non int feature result")
530 }
531 }
532 variations[fmt.Sprint(v)]++
533 }
534 }
535
View as plain text