diff --git a/backend/server/server_test.go b/backend/server/server_test.go
index c96f0ec..8a041f0 100644
--- a/backend/server/server_test.go
+++ b/backend/server/server_test.go
@@ -141,7 +141,7 @@ func TestUpdateReleaseVersion(t *testing.T) {
 func TestGithubRedirects(t *testing.T) {
 	// Set up
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
+	defer shared.RunTestServer()()
 
 	// Check the redirects
 	client := &http.Client{
diff --git a/client/client_test.go b/client/client_test.go
index d8bc313..460aafb 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -21,6 +21,11 @@ import (
 
 // TODO: Change this to only start the server once for this entire file
 
+func TestMain(m *testing.M) {
+	defer shared.RunTestServer()()
+	m.Run()
+}
+
 func RunInteractiveBashCommands(t *testing.T, script string) string {
 	out, err := RunInteractiveBashCommandsWithoutStrictMode(t, "set -emo pipefail\n"+script)
 	if err != nil {
@@ -51,7 +56,6 @@ func RunInteractiveBashCommandsWithoutStrictMode(t *testing.T, script string) (s
 func TestIntegration(t *testing.T) {
 	// Set up
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
 
 	// Run the test
 	testIntegration(t)
@@ -60,7 +64,6 @@ func TestIntegration(t *testing.T) {
 func TestIntegrationWithNewDevice(t *testing.T) {
 	// Set up
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
 
 	// Run the test
 	userSecret := testIntegration(t)
@@ -148,7 +151,6 @@ func TestIntegrationWithNewDevice(t *testing.T) {
 	}
 
 	// Finally, test the export command
-	waitForBackgroundSavesToComplete(t)
 	out = RunInteractiveBashCommands(t, `hishtory export`)
 	if strings.Contains(out, "thisisnotrecorded") {
 		t.Fatalf("hishtory export contains a command that should not have been recorded, out=%#v", out)
@@ -246,7 +248,6 @@ echo thisisrecorded`)
 func TestAdvancedQuery(t *testing.T) {
 	// Set up
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
 
 	// Install hishtory
 	userSecret := installHishtory(t, "")
@@ -421,7 +422,6 @@ hishtory disable`)
 func TestUpdate(t *testing.T) {
 	// Set up
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
 	userSecret := installHishtory(t, "")
 
 	// Check the status command
@@ -446,7 +446,6 @@ func TestUpdate(t *testing.T) {
 func TestRepeatedCommandThenQuery(t *testing.T) {
 	// Set up
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
 	userSecret := installHishtory(t, "")
 
 	// Check the status command
@@ -483,7 +482,6 @@ echo mycommand-3`)
 func TestRepeatedCommandAndQuery(t *testing.T) {
 	// Set up
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
 	userSecret := installHishtory(t, "")
 
 	// Check the status command
@@ -509,7 +507,6 @@ func TestRepeatedCommandAndQuery(t *testing.T) {
 func TestRepeatedEnableDisable(t *testing.T) {
 	// Set up
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
 	installHishtory(t, "")
 
 	// Run a command many times
@@ -542,7 +539,6 @@ hishtory enable`, i))
 func TestExcludeHiddenCommand(t *testing.T) {
 	// Set up
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
 	installHishtory(t, "")
 
 	RunInteractiveBashCommands(t, `echo hello1
@@ -595,7 +591,6 @@ func waitForBackgroundSavesToComplete(t *testing.T) {
 }
 
 func hishtoryQuery(t *testing.T, query string) string {
-	waitForBackgroundSavesToComplete(t)
 	return RunInteractiveBashCommands(t, "hishtory query "+query)
 }
 
@@ -616,7 +611,6 @@ func manuallySubmitHistoryEntry(t *testing.T, userSecret string, entry data.Hist
 func TestHishtoryBackgroundSaving(t *testing.T) {
 	// Setup
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
 
 	// Test install with an unset HISHTORY_TEST var so that we save in the background (this is likely to be flakey!)
 	out := RunInteractiveBashCommands(t, `unset HISHTORY_TEST
@@ -650,6 +644,7 @@ echo foo`)
 	}
 
 	// Test querying for all commands
+	waitForBackgroundSavesToComplete(t)
 	out = hishtoryQuery(t, "")
 	expected := []string{"echo foo", "ls /a"}
 	for _, item := range expected {
@@ -659,6 +654,7 @@ echo foo`)
 	}
 
 	// Test querying for a specific command
+	waitForBackgroundSavesToComplete(t)
 	out = hishtoryQuery(t, "foo")
 	if !strings.Contains(out, "echo foo") {
 		t.Fatalf("output doesn't contain the expected item, out=%#v", out)
diff --git a/client/lib/lib_test.go b/client/lib/lib_test.go
index 9a3f368..1c88ae9 100644
--- a/client/lib/lib_test.go
+++ b/client/lib/lib_test.go
@@ -14,7 +14,7 @@ import (
 
 func TestSetup(t *testing.T) {
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
+	defer shared.RunTestServer()()
 	homedir, err := os.UserHomeDir()
 	shared.Check(t, err)
 	if _, err := os.Stat(path.Join(homedir, shared.HISHTORY_PATH, shared.CONFIG_PATH)); err == nil {
@@ -33,7 +33,7 @@ func TestSetup(t *testing.T) {
 
 func TestBuildHistoryEntry(t *testing.T) {
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
+	defer shared.RunTestServer()()
 	shared.Check(t, Setup([]string{}))
 
 	// Test building an actual entry
@@ -68,7 +68,7 @@ func TestBuildHistoryEntry(t *testing.T) {
 
 func TestGetUserSecret(t *testing.T) {
 	defer shared.BackupAndRestore(t)()
-	defer shared.RunTestServer(t)()
+	defer shared.RunTestServer()()
 	shared.Check(t, Setup([]string{}))
 	secret1, err := GetUserSecret()
 	shared.Check(t, err)
diff --git a/shared/testutils.go b/shared/testutils.go
index 88d0037..e8ebb59 100644
--- a/shared/testutils.go
+++ b/shared/testutils.go
@@ -2,6 +2,7 @@ package shared
 
 import (
 	"bytes"
+	"fmt"
 	"os"
 	"os/exec"
 	"path"
@@ -41,21 +42,21 @@ func BackupAndRestore(t *testing.T) func() {
 	}
 }
 
-func buildServer(t *testing.T) {
+func buildServer() {
 	for {
 		wd, err := os.Getwd()
 		if err != nil {
-			t.Fatalf("failed to getwd: %v", err)
+			panic(fmt.Sprintf("failed to getwd: %v", err))
 		}
 		if strings.HasSuffix(wd, "/hishtory") {
 			break
 		}
 		err = os.Chdir("../")
 		if err != nil {
-			t.Fatalf("failed to chdir: %v", err)
+			panic(fmt.Sprintf("failed to chdir: %v", err))
 		}
 		if wd == "/" {
-			t.Fatalf("failed to cd into hishtory dir!")
+			panic("failed to cd into hishtory dir!")
 		}
 	}
 	cmd := exec.Command("go", "build", "-o", "/tmp/server", "backend/server/server.go")
@@ -65,18 +66,18 @@ func buildServer(t *testing.T) {
 	cmd.Stderr = &stderr
 	err := cmd.Start()
 	if err != nil {
-		t.Fatalf("failed to start to build server: %v, stderr=%#v, stdout=%#v", err, stderr.String(), stdout.String())
+		panic(fmt.Sprintf("failed to start to build server: %v, stderr=%#v, stdout=%#v", err, stderr.String(), stdout.String()))
 	}
 	err = cmd.Wait()
 	if err != nil {
 		wd, _ := os.Getwd()
-		t.Fatalf("failed to build server: %v, wd=%#v, stderr=%#v, stdout=%#v", err, wd, stderr.String(), stdout.String())
+		panic(fmt.Sprintf("failed to build server: %v, wd=%#v, stderr=%#v, stdout=%#v", err, wd, stderr.String(), stdout.String()))
 	}
 }
 
-func RunTestServer(t *testing.T) func() {
+func RunTestServer() func() {
 	os.Setenv("HISHTORY_SERVER", "http://localhost:8080")
-	buildServer(t)
+	buildServer()
 	cmd := exec.Command("/tmp/server")
 	var stdout bytes.Buffer
 	cmd.Stdout = &stdout
@@ -84,20 +85,19 @@ func RunTestServer(t *testing.T) func() {
 	cmd.Stderr = &stderr
 	err := cmd.Start()
 	if err != nil {
-		t.Fatalf("failed to start server: %v", err)
+		panic(fmt.Sprintf("failed to start server: %v", err))
 	}
-	// TODO: Optimize this by streaming stdout and waiting until we see the "listening ..." message
-	time.Sleep(time.Second * 3)
+	time.Sleep(time.Second * 5)
 	go func() {
 		_ = cmd.Wait()
 	}()
 	return func() {
 		err := cmd.Process.Kill()
 		if err != nil && err.Error() != "os: process already finished" {
-			t.Fatalf("failed to kill process: %v", err)
+			panic(fmt.Sprintf("failed to kill server process: %v", err))
 		}
 		if strings.Contains(stderr.String()+stdout.String(), "failed to") {
-			t.Fatalf("server failed to do something: stderr=%#v, stdout=%#v", stderr.String(), stdout.String())
+			panic(fmt.Sprintf("server failed to do something: stderr=%#v, stdout=%#v", stderr.String(), stdout.String()))
 		}
 		// fmt.Printf("stderr=%#v, stdout=%#v\n", stderr.String(), stdout.String())
 	}