...

Source file src/github.com/growthbook/growthbook-golang/experiment_test.go

Documentation: github.com/growthbook/growthbook-golang

     1  package growthbook
     2  
     3  import (
     4  	"fmt"
     5  	"reflect"
     6  	"regexp"
     7  	"testing"
     8  )
     9  
    10  type trackCall struct {
    11  	experiment *Experiment
    12  	result     *Result
    13  }
    14  
    15  type tracker struct {
    16  	calls []trackCall
    17  	cb    func(experiment *Experiment, result *Result)
    18  }
    19  
    20  func track() *tracker {
    21  	t := tracker{[]trackCall{}, nil}
    22  	t.cb = func(experiment *Experiment, result *Result) {
    23  		t.calls = append(t.calls, trackCall{experiment, result})
    24  	}
    25  	return &t
    26  }
    27  
    28  func TestExperimentTracking(t *testing.T) {
    29  	context := NewContext().
    30  		WithUserAttributes(Attributes{"id": "1"})
    31  
    32  	tr := track()
    33  	gb := New(context).WithTrackingCallback(tr.cb)
    34  
    35  	exp1 := NewExperiment("my-tracked-test").WithVariations(0, 1)
    36  	exp2 := NewExperiment("my-other-tracked-test").WithVariations(0, 1)
    37  
    38  	res1 := gb.Run(exp1)
    39  	gb.Run(exp1)
    40  	gb.Run(exp1)
    41  	res4 := gb.Run(exp2)
    42  	context = context.WithUserAttributes(Attributes{"id": "2"})
    43  	res5 := gb.Run(exp2)
    44  
    45  	if len(tr.calls) != 3 {
    46  		t.Errorf("expected 3 calls to tracking callback, got %d", len(tr.calls))
    47  	} else {
    48  		if !reflect.DeepEqual(tr.calls[0], trackCall{exp1, res1}) {
    49  			t.Errorf("unexpected callback result")
    50  		}
    51  		if !reflect.DeepEqual(tr.calls[1], trackCall{exp2, res4}) {
    52  			t.Errorf("unexpected callback result")
    53  		}
    54  		if !reflect.DeepEqual(tr.calls[2], trackCall{exp2, res5}) {
    55  			t.Errorf("unexpected callback result")
    56  		}
    57  	}
    58  }
    59  
    60  func TestExperimentForcesVariationFromOverrides(t *testing.T) {
    61  	forceVal := 1
    62  	context := NewContext().
    63  		WithOverrides(ExperimentOverrides{
    64  			"forced-test": &ExperimentOverride{
    65  				Force: &forceVal,
    66  			}})
    67  	gb := New(context).
    68  		WithAttributes(Attributes{"id": "6"})
    69  
    70  	res := gb.Run(NewExperiment("forced-test").WithVariations(0, 1))
    71  
    72  	if res.VariationID != 1 {
    73  		t.Error("expected variation ID 1, got", res.VariationID)
    74  	}
    75  	if res.InExperiment != true {
    76  		t.Error("expected InExperiment to be true")
    77  	}
    78  	if res.HashUsed != false {
    79  		t.Error("expected HashUsed to be false")
    80  	}
    81  }
    82  
    83  func TestExperimentCoverageFromOverrides(t *testing.T) {
    84  	overrideVal := 0.01
    85  	context := NewContext().
    86  		WithOverrides(ExperimentOverrides{
    87  			"my-test": &ExperimentOverride{
    88  				Coverage: &overrideVal,
    89  			}})
    90  	gb := New(context).
    91  		WithAttributes(Attributes{"id": "1"})
    92  
    93  	res := gb.Run(NewExperiment("my-test").WithVariations(0, 1))
    94  
    95  	if res.VariationID != 0 {
    96  		t.Error("expected variation ID 0, got", res.VariationID)
    97  	}
    98  	if res.InExperiment != false {
    99  		t.Error("expected InExperiment to be false")
   100  	}
   101  }
   102  
   103  func TestExperimentDoesNotTrackWhenForcedWithOverrides(t *testing.T) {
   104  	context := NewContext().
   105  		WithUserAttributes(Attributes{"id": "6"})
   106  	tr := track()
   107  	gb := New(context).WithTrackingCallback(tr.cb)
   108  	exp := NewExperiment("forced-test").WithVariations(0, 1)
   109  
   110  	forceVal := 1
   111  	context = context.WithOverrides(ExperimentOverrides{
   112  		"forced-test": &ExperimentOverride{Force: &forceVal},
   113  	})
   114  
   115  	gb.Run(exp)
   116  
   117  	if len(tr.calls) != 0 {
   118  		t.Error("expected 0 calls to tracking callback, got ", len(tr.calls))
   119  	}
   120  }
   121  
   122  func TestExperimentURLFromOverrides(t *testing.T) {
   123  	urlRe := regexp.MustCompile(`^\/path`)
   124  	context := NewContext().
   125  		WithUserAttributes(Attributes{"id": "1"}).
   126  		WithOverrides(ExperimentOverrides{
   127  			"my-test": &ExperimentOverride{URL: urlRe},
   128  		})
   129  	gb := New(context)
   130  
   131  	if gb.Run(NewExperiment("my-test").WithVariations(0, 1)).InExperiment != false {
   132  		t.Error("expected InExperiment to be false")
   133  	}
   134  }
   135  
   136  func TestExperimentFiltersUserGroups(t *testing.T) {
   137  	context := NewContext().
   138  		WithUserAttributes(Attributes{"id": "123"}).
   139  		WithGroups(map[string]bool{
   140  			"alpha":    true,
   141  			"beta":     true,
   142  			"internal": false,
   143  			"qa":       false,
   144  		})
   145  	gb := New(context)
   146  
   147  	exp := NewExperiment("my-test").
   148  		WithVariations(0, 1).
   149  		WithGroups("internal", "qa")
   150  	if gb.Run(exp).InExperiment != false {
   151  		t.Error("1: expected InExperiment to be false")
   152  	}
   153  
   154  	exp = NewExperiment("my-test").
   155  		WithVariations(0, 1).
   156  		WithGroups("internal", "qa", "beta")
   157  	if gb.Run(exp).InExperiment != true {
   158  		t.Error("2: expected InExperiment to be true")
   159  	}
   160  
   161  	exp = NewExperiment("my-test").
   162  		WithVariations(0, 1)
   163  	if gb.Run(exp).InExperiment != true {
   164  		t.Error("3: expected InExperiment to be true")
   165  	}
   166  }
   167  
   168  func TestExperimentSetsAttributes(t *testing.T) {
   169  	attributes := Attributes{
   170  		"id":      "1",
   171  		"browser": "firefox",
   172  	}
   173  	gb := New(nil).WithAttributes(attributes)
   174  
   175  	if !reflect.DeepEqual(gb.Attributes(), attributes) {
   176  		t.Error("expected attributes to match")
   177  	}
   178  }
   179  
   180  func TestExperimentCustomIncludeCallback(t *testing.T) {
   181  	context := NewContext().
   182  		WithUserAttributes(Attributes{"id": "1"})
   183  	gb := New(context)
   184  
   185  	exp := NewExperiment("my-test").
   186  		WithVariations(0, 1).
   187  		WithIncludeFunction(func() bool { return false })
   188  
   189  	if gb.Run(exp).InExperiment != false {
   190  		t.Error("expected InExperiment to be false")
   191  	}
   192  }
   193  
   194  func TestExperimentTrackingSkippedWhenContextDisabled(t *testing.T) {
   195  	context := NewContext().
   196  		WithUserAttributes(Attributes{"id": "1"}).
   197  		WithEnabled(false)
   198  	tr := track()
   199  	gb := New(context).WithTrackingCallback(tr.cb)
   200  
   201  	gb.Run(NewExperiment("disabled-test").WithVariations(0, 1))
   202  
   203  	if len(tr.calls) != 0 {
   204  		t.Errorf("expected 0 calls to tracking callback, got %d", len(tr.calls))
   205  	}
   206  }
   207  
   208  func TestExperimentQuerystringForceDisablsTracking(t *testing.T) {
   209  	context := NewContext().
   210  		WithUserAttributes(Attributes{"id": "1"}).
   211  		WithURL(mustParseUrl("http://example.com?forced-test-qs=1"))
   212  	tr := track()
   213  	gb := New(context).WithTrackingCallback(tr.cb)
   214  
   215  	gb.Run(NewExperiment("forced-test-qs").WithVariations(0, 1))
   216  
   217  	if len(tr.calls) != 0 {
   218  		t.Errorf("expected 0 calls to tracking callback, got %d", len(tr.calls))
   219  	}
   220  }
   221  
   222  func TestExperimentURLTargeting(t *testing.T) {
   223  	context := NewContext().
   224  		WithUserAttributes(Attributes{"id": "1"}).
   225  		WithURL(mustParseUrl("http://example.com"))
   226  	gb := New(context)
   227  
   228  	exp := NewExperiment("my-test").
   229  		WithVariations(0, 1).
   230  		WithURL(regexp.MustCompile("^/post/[0-9]+"))
   231  
   232  	check := func(icase int, e *Experiment, inExperiment bool, value interface{}) {
   233  		result := gb.Run(e)
   234  		if result.InExperiment != inExperiment {
   235  			t.Errorf("%d: expected InExperiment = %v, got %v",
   236  				icase, inExperiment, result.InExperiment)
   237  		}
   238  		if !reflect.DeepEqual(result.Value, value) {
   239  			t.Errorf("%d: expected value = %v, got %v",
   240  				icase, value, result.Value)
   241  		}
   242  	}
   243  
   244  	check(1, exp, false, 0)
   245  
   246  	context = context.WithURL(mustParseUrl("http://example.com/post/123"))
   247  	check(2, exp, true, 1)
   248  
   249  	exp.URL = regexp.MustCompile("http://example.com/post/[0-9]+")
   250  	check(3, exp, true, 1)
   251  }
   252  
   253  func TestExperimentIgnoresDraftExperiments(t *testing.T) {
   254  	context := NewContext().
   255  		WithUserAttributes(Attributes{"id": "1"})
   256  	gb := New(context)
   257  
   258  	exp := NewExperiment("my-test").
   259  		WithStatus(DraftStatus).
   260  		WithVariations(0, 1)
   261  
   262  	res1 := gb.Run(exp)
   263  	context = context.WithURL(mustParseUrl("http://example.com/?my-test=1"))
   264  	res2 := gb.Run(exp)
   265  
   266  	if res1.InExperiment != false {
   267  		t.Error("1: expected InExperiment to be false")
   268  	}
   269  	if res1.HashUsed != false {
   270  		t.Error("1: expected HashUsed to be false")
   271  	}
   272  	if res1.Value != 0 {
   273  		t.Errorf("1: expected Value to be 0, got %v", res1.Value)
   274  	}
   275  
   276  	if res2.InExperiment != true {
   277  		t.Error("2: expected InExperiment to be true")
   278  	}
   279  	if res2.HashUsed != false {
   280  		t.Error("2: expected HashUsed to be false")
   281  	}
   282  	if res2.Value != 1 {
   283  		t.Errorf("2: expected Value to be 1, got %v", res2.Value)
   284  	}
   285  }
   286  
   287  func TestExperimentIgnoresStoppedExperimentsUnlessForced(t *testing.T) {
   288  	context := NewContext().
   289  		WithUserAttributes(Attributes{"id": "1"})
   290  	gb := New(context)
   291  
   292  	expLose := NewExperiment("my-test").
   293  		WithStatus(StoppedStatus).
   294  		WithVariations(0, 1, 2)
   295  	expWin := NewExperiment("my-test").
   296  		WithStatus(StoppedStatus).
   297  		WithVariations(0, 1, 2).
   298  		WithForce(2)
   299  
   300  	res1 := gb.Run(expLose)
   301  	res2 := gb.Run(expWin)
   302  
   303  	if res1.InExperiment != false {
   304  		t.Error("1: expected InExperiment to be false")
   305  	}
   306  	if res1.HashUsed != false {
   307  		t.Error("1: expected HashUsed to be false")
   308  	}
   309  	if res1.Value != 0 {
   310  		t.Errorf("1: expected Value to be 0, got %v", res1.Value)
   311  	}
   312  
   313  	if res2.InExperiment != true {
   314  		t.Error("2: expected InExperiment to be true")
   315  	}
   316  	if res2.HashUsed != false {
   317  		t.Error("2: expected HashUsed to be false")
   318  	}
   319  	if res2.Value != 2 {
   320  		t.Errorf("2: expected Value to be 2, got %v", res2.Value)
   321  	}
   322  }
   323  
   324  func TestExperimentDoesEvenWeighting(t *testing.T) {
   325  	context := NewContext()
   326  	gb := New(context)
   327  
   328  	// Full coverage
   329  	exp := NewExperiment("my-test").WithVariations(0, 1)
   330  	variations := map[string]int{
   331  		"0":  0,
   332  		"1":  0,
   333  		"-1": 0,
   334  	}
   335  	countVariations(t, context, gb, exp, 1000, variations)
   336  	if variations["0"] != 503 {
   337  		t.Errorf("1: expected variations[\"0\"] to be 503, got %v", variations["0"])
   338  	}
   339  
   340  	// Reduced coverage
   341  	exp = exp.WithCoverage(0.4)
   342  	variations = map[string]int{
   343  		"0":  0,
   344  		"1":  0,
   345  		"-1": 0,
   346  	}
   347  	countVariations(t, context, gb, exp, 10000, variations)
   348  	if variations["0"] != 2044 {
   349  		t.Errorf("2: expected variations[\"0\"] to be 2044, got %v", variations["0"])
   350  	}
   351  	if variations["1"] != 1980 {
   352  		t.Errorf("2: expected variations[\"1\"] to be 1980, got %v", variations["0"])
   353  	}
   354  	if variations["-1"] != 5976 {
   355  		t.Errorf("2: expected variations[\"0\"] to be 5976, got %v", variations["0"])
   356  	}
   357  
   358  	// 3-way
   359  	exp = exp.WithCoverage(0.6).WithVariations(0, 1, 2)
   360  	variations = map[string]int{
   361  		"0":  0,
   362  		"1":  0,
   363  		"2":  0,
   364  		"-1": 0,
   365  	}
   366  	countVariations(t, context, gb, exp, 10000, variations)
   367  	expected := map[string]int{
   368  		"-1": 3913,
   369  		"0":  2044,
   370  		"1":  2000,
   371  		"2":  2043,
   372  	}
   373  	if !reflect.DeepEqual(variations, expected) {
   374  		t.Errorf("3: expected variations counts %#v, git %#v", expected, variations)
   375  	}
   376  }
   377  
   378  func TestExperimentForcesMultipleVariationsAtOnce(t *testing.T) {
   379  	context := NewContext().
   380  		WithUserAttributes(Attributes{"id": "1"})
   381  	gb := New(context)
   382  
   383  	exp := NewExperiment("my-test").
   384  		WithVariations(0, 1)
   385  
   386  	res1 := gb.Run(exp)
   387  	commonCheck(t, 1, res1, true, true, 1)
   388  
   389  	gb = gb.WithForcedVariations(ForcedVariationsMap{
   390  		"my-test": 0,
   391  	})
   392  	res2 := gb.Run(exp)
   393  	commonCheck(t, 2, res2, true, false, 0)
   394  
   395  	gb = gb.WithForcedVariations(nil)
   396  	res3 := gb.Run(exp)
   397  	commonCheck(t, 3, res3, true, true, 1)
   398  }
   399  
   400  func TestExperimentOnceForcesAllVariationsInQAMode(t *testing.T) {
   401  	context := NewContext().
   402  		WithUserAttributes(Attributes{"id": "1"}).
   403  		WithQAMode(true)
   404  	gb := New(context)
   405  
   406  	exp := NewExperiment("my-test").
   407  		WithVariations(0, 1)
   408  
   409  	res1 := gb.Run(exp)
   410  	commonCheck(t, 1, res1, false, false, 0)
   411  
   412  	// Still works if explicitly forced
   413  	context = context.WithForcedVariations(ForcedVariationsMap{"my-test": 1})
   414  	res2 := gb.Run(exp)
   415  	commonCheck(t, 2, res2, true, false, 1)
   416  
   417  	// Works if the experiment itself is forced
   418  	exp2 := NewExperiment("my-test-2").WithVariations(0, 1).WithForce(1)
   419  	res3 := gb.Run(exp2)
   420  	commonCheck(t, 3, res3, true, false, 1)
   421  }
   422  
   423  func TestExperimentFiresSubscriptionsCorrectly(t *testing.T) {
   424  	context := NewContext().
   425  		WithUserAttributes(Attributes{"id": "1"})
   426  	gb := New(context)
   427  
   428  	fired := false
   429  	checkFired := func(icase int, f bool) {
   430  		if fired != f {
   431  			t.Errorf("%d: expected fired to be %v", icase, f)
   432  		}
   433  	}
   434  
   435  	unsubscriber := gb.Subscribe(func(experiment *Experiment, result *Result) {
   436  		fired = true
   437  	})
   438  	checkFired(1, false)
   439  
   440  	exp := NewExperiment("my-test").WithVariations(0, 1)
   441  
   442  	// Should fire when user is put in an experiment
   443  	gb.Run(exp)
   444  	checkFired(2, true)
   445  
   446  	// Does not fire if nothing has changed
   447  	fired = false
   448  	gb.Run(exp)
   449  	checkFired(3, false)
   450  
   451  	// Does not fire after unsubscribed
   452  	unsubscriber()
   453  	exp2 := NewExperiment("other-test").WithVariations(0, 1)
   454  	gb.Run(exp2)
   455  	checkFired(4, false)
   456  }
   457  
   458  func TestExperimentStoresAssignedVariations(t *testing.T) {
   459  	context := NewContext().
   460  		WithUserAttributes(Attributes{"id": "1"})
   461  	gb := New(context)
   462  	gb.Run(NewExperiment("my-test").WithVariations(0, 1))
   463  	gb.Run(NewExperiment("my-test-3").WithVariations(0, 1))
   464  
   465  	assignedVars := gb.GetAllResults()
   466  
   467  	if len(assignedVars) != 2 {
   468  		t.Errorf("expected len(assignedVars) to be 2, got %d", len(assignedVars))
   469  	}
   470  	if assignedVars["my-test"].Result.VariationID != 1 {
   471  		t.Errorf("expected assignedVars[\"my-test\"] to be 1, got %d",
   472  			assignedVars["my-test"].Result.VariationID)
   473  	}
   474  	if assignedVars["my-test-3"].Result.VariationID != 0 {
   475  		t.Errorf("expected assignedVars[\"my-test-3\"] to be 0, got %d",
   476  			assignedVars["my-test-3"].Result.VariationID)
   477  	}
   478  }
   479  
   480  func TestExperimentDoesNotHaveBiasWhenUsingNamespaces(t *testing.T) {
   481  	context := NewContext().
   482  		WithUserAttributes(Attributes{"id": "1"})
   483  	gb := New(context)
   484  
   485  	variations := map[string]int{
   486  		"0":  0,
   487  		"1":  0,
   488  		"-1": 0,
   489  	}
   490  
   491  	exp := NewExperiment("my-test").
   492  		WithVariations(0, 1).
   493  		WithNamespace(&Namespace{"namespace", 0.0, 0.5})
   494  	countVariations(t, context, gb, exp, 10000, variations)
   495  
   496  	expected := map[string]int{
   497  		"-1": 4973,
   498  		"0":  2538,
   499  		"1":  2489,
   500  	}
   501  	if !reflect.DeepEqual(variations, expected) {
   502  		t.Errorf("expected variations counts %#v, git %#v", expected, variations)
   503  	}
   504  }
   505  
   506  func commonCheck(t *testing.T, icase int, res *Result,
   507  	inExperiment bool, hashUsed bool, value FeatureValue) {
   508  	if res.InExperiment != inExperiment {
   509  		t.Errorf("%d: expected InExperiment to be %v", icase, inExperiment)
   510  	}
   511  	if res.HashUsed != hashUsed {
   512  		t.Errorf("%d: expected HashUsed to be %v", icase, hashUsed)
   513  	}
   514  	if res.Value != value {
   515  		t.Errorf("3: expected Value to be %#v, got %#v", value, res.Value)
   516  	}
   517  }
   518  
   519  func countVariations(t *testing.T, context *Context, gb *GrowthBook,
   520  	exp *Experiment, runs int, variations map[string]int) {
   521  	for i := 0; i < runs; i++ {
   522  		context = context.WithUserAttributes(Attributes{"id": fmt.Sprint(i)})
   523  		res := gb.Run(exp)
   524  		v := -1
   525  		ok := false
   526  		if res.InExperiment {
   527  			v, ok = res.Value.(int)
   528  			if !ok {
   529  				t.Error("non int feature result")
   530  			}
   531  		}
   532  		variations[fmt.Sprint(v)]++
   533  	}
   534  }
   535  

View as plain text