From 87105ec960b7f6d0d87aaba3ad3dd674fb3ddc76 Mon Sep 17 00:00:00 2001 From: Yevhenii Voevodin Date: Mon, 29 May 2017 14:29:33 +0300 Subject: [PATCH] Add exec-agent rest service tests --- agents/go-agents/core/process/common.go | 10 +- agents/go-agents/core/process/events.go | 5 +- agents/go-agents/core/process/process_test.go | 105 +------ .../core/process/processtest/test_utils.go | 116 +++++++ .../exec-agent/exec/rest_service_test.go | 288 ++++++++++++++++++ 5 files changed, 429 insertions(+), 95 deletions(-) create mode 100644 agents/go-agents/core/process/processtest/test_utils.go create mode 100644 agents/go-agents/exec-agent/exec/rest_service_test.go diff --git a/agents/go-agents/core/process/common.go b/agents/go-agents/core/process/common.go index 1cf63674fd..f2419b9f0c 100644 --- a/agents/go-agents/core/process/common.go +++ b/agents/go-agents/core/process/common.go @@ -16,13 +16,13 @@ import ( "time" ) -// LogKind represents kind of source of log line - stdout or stderr +// LogKind represents kind of source of log line - stdout or stderr. type LogKind int const ( - // StdoutKind match logs produced by stdout of a process + // StdoutKind match logs produced by stdout of a process. StdoutKind LogKind = iota - // StderrKind match logs produced by stderr of a process + // StderrKind match logs produced by stderr of a process. StderrKind ) @@ -30,7 +30,7 @@ const ( // Single date format keeps API consistent var DateTimeFormat = time.RFC3339Nano -// LogMessage represents single log entry with timestamp and source of log +// LogMessage represents single log entry with timestamp and source of log. type LogMessage struct { Kind LogKind `json:"kind"` Time time.Time `json:"time"` @@ -40,7 +40,7 @@ type LogMessage struct { // ParseTime parses string into Time. // If time string is empty, then time provided as an argument is returned. // If time string is invalid, then appropriate error is returned. -// If time string is valid then parsed time is returned +// If time string is valid then parsed time is returned. func ParseTime(timeStr string, defTime time.Time) (time.Time, error) { if timeStr == "" { return defTime, nil diff --git a/agents/go-agents/core/process/events.go b/agents/go-agents/core/process/events.go index 08fa9e0d0c..16bc0c715a 100644 --- a/agents/go-agents/core/process/events.go +++ b/agents/go-agents/core/process/events.go @@ -28,7 +28,7 @@ type Event interface { Type() string } -// EventConsumer provides ability to handle process events. +// EventConsumer is a process events client. type EventConsumer interface { Accept(event Event) } @@ -42,6 +42,7 @@ type StartedEvent struct { CommandLine string `json:"commandLine"` } +// Type returns StartedEventType. func (se *StartedEvent) Type() string { return StartedEventType } func newStartedEvent(mp MachineProcess) *StartedEvent { @@ -64,6 +65,7 @@ type DiedEvent struct { ExitCode int `json:"exitCode"` } +// Type returns DiedEventType. func (de *DiedEvent) Type() string { return DiedEventType } func newDiedEvent(mp MachineProcess) *DiedEvent { @@ -86,6 +88,7 @@ type OutputEvent struct { _type string } +// Type returns one of StdoutEventType, StderrEventType. func (se *OutputEvent) Type() string { return se._type } func newStderrEvent(pid uint64, text string, when time.Time) Event { diff --git a/agents/go-agents/core/process/process_test.go b/agents/go-agents/core/process/process_test.go index 56e1e1b566..78423eae01 100644 --- a/agents/go-agents/core/process/process_test.go +++ b/agents/go-agents/core/process/process_test.go @@ -21,7 +21,7 @@ import ( "fmt" "github.com/eclipse/che/agents/go-agents/core/process" - "sync" + "github.com/eclipse/che/agents/go-agents/core/process/processtest" ) const ( @@ -193,7 +193,7 @@ func TestReadProcessLogs(t *testing.T) { } func TestLogsAreNotWrittenIfLogsDirIsNotSet(t *testing.T) { - p := doStartAndWaitTestProcess(testCmd, "", &eventsCaptor{deathEventType: process.DiedEventType}, t) + p := doStartAndWaitTestProcess(testCmd, "", processtest.NewEventsCaptor(process.DiedEventType), t) _, err := process.ReadAllLogs(p.Pid) if err == nil { @@ -207,7 +207,7 @@ func TestLogsAreNotWrittenIfLogsDirIsNotSet(t *testing.T) { } func TestAllProcessLifeCycleEventsArePublished(t *testing.T) { - eventsCaptor := &eventsCaptor{deathEventType: process.DiedEventType} + eventsCaptor := processtest.NewEventsCaptor(process.DiedEventType) doStartAndWaitTestProcess("printf \"first_line\nsecond_line\"", "", eventsCaptor, t) expected := []string{ @@ -216,18 +216,19 @@ func TestAllProcessLifeCycleEventsArePublished(t *testing.T) { process.StdoutEventType, process.DiedEventType, } - checkEventsOrder(t, eventsCaptor.events, expected...) + checkEventsOrder(t, eventsCaptor.Events(), expected...) } func TestProcessExitCodeIs0IfFinishedOk(t *testing.T) { - captor := &eventsCaptor{deathEventType: process.DiedEventType} + captor := processtest.NewEventsCaptor(process.DiedEventType) p := doStartAndWaitTestProcess("echo test", "", captor, t) if p.ExitCode != 0 { t.Fatalf("Expected process exit code to be 0, but it is %d", p.ExitCode) } - if diedEvent, ok := captor.events[len(captor.events)-1].(*process.DiedEvent); !ok { + events := captor.Events() + if diedEvent, ok := events[len(events)-1].(*process.DiedEvent); !ok { t.Fatalf("Expected last captured event to be process died event, but it is %s", diedEvent.Type()) } else if diedEvent.ExitCode != 0 { t.Fatalf("Expected process died event exit code to be 0, but it is %d", diedEvent.ExitCode) @@ -235,7 +236,7 @@ func TestProcessExitCodeIs0IfFinishedOk(t *testing.T) { } func TestProcessExitCodeIsNot0IfFinishedNotOk(t *testing.T) { - captor := &eventsCaptor{deathEventType: process.DiedEventType} + captor := processtest.NewEventsCaptor(process.DiedEventType) // starting non-existing command(hopefully) p := doStartAndWaitTestProcess("test-process-cmd-"+randomName(10), "", captor, t) @@ -243,7 +244,8 @@ func TestProcessExitCodeIsNot0IfFinishedNotOk(t *testing.T) { t.Fatalf("Expected process exit code to be > 0, but it is %d", p.ExitCode) } - if diedEvent, ok := captor.events[len(captor.events)-1].(*process.DiedEvent); !ok { + events := captor.Events() + if diedEvent, ok := events[len(events)-1].(*process.DiedEvent); !ok { t.Fatalf("Expected last captured event to be process died event, but it is %s", diedEvent.Type()) } else if diedEvent.ExitCode <= 0 { t.Fatalf("Expected process died event exit code to be > 0, but it is %d", diedEvent.ExitCode) @@ -266,19 +268,19 @@ func failIfEventTypeIsDifferent(t *testing.T, event process.Event, expectedType } func startAndWaitTestProcess(cmd string, t *testing.T) process.MachineProcess { - p := doStartAndWaitTestProcess(cmd, "", &eventsCaptor{deathEventType: process.DiedEventType}, t) + p := doStartAndWaitTestProcess(cmd, "", processtest.NewEventsCaptor(process.DiedEventType), t) return p } func startAndWaitTestProcessWritingLogsToTmpDir(cmd string, t *testing.T) process.MachineProcess { - p := doStartAndWaitTestProcess(cmd, tmpFile(), &eventsCaptor{deathEventType: process.DiedEventType}, t) + p := doStartAndWaitTestProcess(cmd, tmpFile(), processtest.NewEventsCaptor(process.DiedEventType), t) return p } -func doStartAndWaitTestProcess(cmd string, logsDir string, eventsCaptor *eventsCaptor, t *testing.T) process.MachineProcess { +func doStartAndWaitTestProcess(cmd string, logsDir string, eventsCaptor *processtest.EventsCaptor, t *testing.T) process.MachineProcess { process.SetLogsDir(logsDir) - eventsCaptor.capture() + eventsCaptor.Capture() pb := process.NewBuilder() pb.CmdName("test") @@ -288,12 +290,12 @@ func doStartAndWaitTestProcess(cmd string, logsDir string, eventsCaptor *eventsC p, err := pb.Start() if err != nil { - eventsCaptor.wait(0) + eventsCaptor.Wait(0) t.Fatal(err) } // wait process for a little while - if ok := <-eventsCaptor.wait(time.Second * 2); !ok { + if ok := <-eventsCaptor.Wait(time.Second * 2); !ok { t.Log("The process doesn't finish its execution in 2 seconds. Trying to kill it") if err := process.Kill(p.Pid); err != nil { t.Logf("Failed to kill process, native pid = %d", p.NativePid) @@ -329,81 +331,6 @@ func randomName(length int) string { return string(bytes) } -// Helps to capture process events and wait for them. -type eventsCaptor struct { - sync.Mutex - - // Result events. - events []process.Event - - // Events channel. Close of this channel considered as immediate interruption, - // to hold until execution completes use captor.wait(timeout) channel. - eventsChan chan process.Event - - // Channel used as internal approach to interrupt capturing. - interruptChan chan bool - - // Captor sends true if finishes reaching deathEventType - // and false if interrupted while waiting for event of deathEventType. - done chan bool - - // The last event after which events capturing stopped. - deathEventType string -} - -func (ec *eventsCaptor) addEvent(e process.Event) { - ec.Lock() - defer ec.Unlock() - ec.events = append(ec.events, e) -} - -func (ec *eventsCaptor) capturedEvents() []process.Event { - ec.Lock() - defer ec.Unlock() - cp := make([]process.Event, len(ec.events)) - copy(cp, ec.events) - return cp -} - -func (ec *eventsCaptor) capture() { - ec.eventsChan = make(chan process.Event) - ec.interruptChan = make(chan bool) - ec.done = make(chan bool) - - go func() { - for { - select { - case event, ok := <-ec.eventsChan: - if ok { - ec.addEvent(event) - if event.Type() == ec.deathEventType { - // death event reached - capturing is done - ec.done <- true - return - } - } else { - // events channel closed interrupt immediately - ec.done <- false - return - } - case <-ec.interruptChan: - close(ec.eventsChan) - } - } - }() -} - -// Waits a timeout and if deadTypeEvent wasn't reached interrupts captor. -func (ec *eventsCaptor) wait(timeout time.Duration) chan bool { - go func() { - <-time.NewTimer(timeout).C - ec.interruptChan <- true - }() - return ec.done -} - -func (ec *eventsCaptor) Accept(e process.Event) { ec.eventsChan <- e } - // A consumer that redirects all the incoming events to the channel. type channelEventConsumer struct { channel chan process.Event diff --git a/agents/go-agents/core/process/processtest/test_utils.go b/agents/go-agents/core/process/processtest/test_utils.go new file mode 100644 index 0000000000..8304b1560b --- /dev/null +++ b/agents/go-agents/core/process/processtest/test_utils.go @@ -0,0 +1,116 @@ +// +// Copyright (c) 2012-2017 Codenvy, S.A. +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// which accompanies this distribution, and is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// Contributors: +// Codenvy, S.A. - initial API and implementation +// + +// Package processtest provides utils for process testing. +package processtest + +import ( + "sync" + "time" + + "github.com/eclipse/che/agents/go-agents/core/process" +) + +// NewEventsCaptor create a new instance of events captor +func NewEventsCaptor(deathEventType string) *EventsCaptor { + return &EventsCaptor{DeathEventType: deathEventType} +} + +// EventsCaptor helps to Capture process events and wait for them. +type EventsCaptor struct { + sync.Mutex + + // Result events. + events []process.Event + + // Events channel. Close of this channel considered as immediate interruption, + // to hold until execution completes use captor.Wait(timeout) channel. + eventsChan chan process.Event + + // Channel used as internal approach to interrupt capturing. + interruptChan chan bool + + // Captor sends true if finishes reaching DeathEventType + // and false if interrupted while waiting for event of DeathEventType. + done chan bool + + // The last event after which events capturing stopped. + DeathEventType string +} + +func (ec *EventsCaptor) addEvent(e process.Event) { + ec.Lock() + defer ec.Unlock() + ec.events = append(ec.events, e) +} + +// Events returns all the captured events. +func (ec *EventsCaptor) Events() []process.Event { + ec.Lock() + defer ec.Unlock() + cp := make([]process.Event, len(ec.events)) + copy(cp, ec.events) + return cp +} + +// Capture starts capturing events, until one of the +// following conditions is met: +// - event of type EventsCaptor.deathEventType received. +// In this case capturing is successful done <- true +// - events channel closed. +// In this case capturing is interrupted done <- false +func (ec *EventsCaptor) Capture() { + ec.eventsChan = make(chan process.Event) + ec.interruptChan = make(chan bool) + ec.done = make(chan bool) + + go func() { + for { + select { + case event, ok := <-ec.eventsChan: + if ok { + ec.addEvent(event) + if event.Type() == ec.DeathEventType { + // death event reached - capturing is done + ec.done <- true + return + } + } else { + // events channel closed interrupt immediately + ec.done <- false + return + } + case <-ec.interruptChan: + close(ec.eventsChan) + } + } + }() +} + +// Waits a timeout and if deadTypeEvent isn't reached interrupts captor. +// Returns done channel if the value received from the channel is true +// then the captor finished capturing successfully catching deathEventType, +// otherwise it was interrupted. +func (ec *EventsCaptor) Wait(timeout time.Duration) chan bool { + go func() { + <-time.NewTimer(timeout).C + ec.interruptChan <- true + }() + return ec.done +} + +// Interrupts capturing immediately, returns done channel. +func (ec *EventsCaptor) Stop() chan bool { + return ec.Wait(0) +} + +// Accept notifies the captor about incoming event. +func (ec *EventsCaptor) Accept(e process.Event) { ec.eventsChan <- e } diff --git a/agents/go-agents/exec-agent/exec/rest_service_test.go b/agents/go-agents/exec-agent/exec/rest_service_test.go new file mode 100644 index 0000000000..6e91b5b868 --- /dev/null +++ b/agents/go-agents/exec-agent/exec/rest_service_test.go @@ -0,0 +1,288 @@ +package exec + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/eclipse/che/agents/go-agents/core/process" + "github.com/eclipse/che/agents/go-agents/core/process/processtest" + "github.com/eclipse/che/agents/go-agents/core/rest" + "net/url" +) + +func TestStartProcessHandlerFunc(t *testing.T) { + command := &process.Command{ + Name: "test", + CommandLine: "echo hello", + Type: "test", + } + req, err := http.NewRequest("POST", "/process", asJSONReader(t, command)) + if err != nil { + t.Fatal(err) + } + rr := httptest.NewRecorder() + + asHTTPHandlerFunc(startProcessHF).ServeHTTP(rr, req) + + if rr.Code != http.StatusOK { + t.Errorf("Expected status code %d but got %d", http.StatusOK, rr.Code) + } + + mp := &process.MachineProcess{} + json.Unmarshal(rr.Body.Bytes(), mp) + failIfDifferent(t, command.Name, mp.Name, "name") + failIfDifferent(t, command.CommandLine, mp.CommandLine, "command-line") + failIfDifferent(t, command.Type, mp.Type, "type") + failIfDifferent(t, -1, mp.ExitCode, "exit-code") + failIfFalse(t, mp.Pid > 0, "Pid > 0") +} + +func TestStartProcessFailsIfCommandIsInvalid(t *testing.T) { + invalidCommands := []*process.Command{ + { + Name: "test", + CommandLine: "", + Type: "test", + }, + { + Name: "", + CommandLine: "echo test", + Type: "test", + }, + } + + for _, command := range invalidCommands { + req, err := http.NewRequest("POST", "/process", asJSONReader(t, command)) + if err != nil { + t.Fatal(err) + } + rr := httptest.NewRecorder() + + asHTTPHandlerFunc(startProcessHF).ServeHTTP(rr, req) + + failIfDifferent(t, http.StatusBadRequest, rr.Code, "status-code") + } +} + +func TestGetsExistingProcess(t *testing.T) { + exp := startAndWaitProcess(t, "echo hello") + + strPid := strconv.Itoa(int(exp.Pid)) + req, err := http.NewRequest("GET", "/process/"+strPid, nil) + if err != nil { + t.Fatal(err) + } + rr := httptest.NewRecorder() + + asHTTPHandlerFunc(getProcessHF, "pid", strPid).ServeHTTP(rr, req) + + failIfDifferent(t, 200, rr.Code, "status-code") + + res := &process.MachineProcess{} + json.Unmarshal(rr.Body.Bytes(), res) + failIfDifferent(t, exp.Pid, res.Pid, "pid") + failIfDifferent(t, exp.Name, res.Name, "name") + failIfDifferent(t, exp.CommandLine, res.CommandLine, "command-line") + failIfDifferent(t, exp.Type, res.Type, "type") + failIfDifferent(t, exp.NativePid, res.NativePid, "native-pid") + failIfDifferent(t, false, res.Alive, "alive") +} + +func TestReturnsNotFoundWhenNoProcess(t *testing.T) { + strPid := "4444" + req, err := http.NewRequest("GET", "/process/"+strPid, nil) + if err != nil { + t.Fatal(err) + } + rr := httptest.NewRecorder() + + asHTTPHandlerFunc(getProcessHF, "pid", strPid).ServeHTTP(rr, req) + + failIfDifferent(t, 404, rr.Code, "status-code") +} + +func TestGetsNoAliveProcesses(t *testing.T) { + startAndWaitProcess(t, "echo test1") + startAndWaitProcess(t, "echo test2") + + req, err := http.NewRequest("GET", "/process", nil) + if err != nil { + t.Fatal(err) + } + rr := httptest.NewRecorder() + + asHTTPHandlerFunc(getProcessesHF).ServeHTTP(rr, req) + + failIfDifferent(t, 200, rr.Code, "status-code") + mps := []process.MachineProcess{} + json.Unmarshal(rr.Body.Bytes(), &mps) + failIfDifferent(t, 0, len(mps), "processes slice len") +} + +func TestGetsProcessLogs(t *testing.T) { + dir, err := ioutil.TempDir(os.TempDir(), "exec-agent-text") + if err != nil { + t.Fatal(err) + } + process.SetLogsDir(dir) + defer process.WipeLogs() + + outputLines := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"} + mp := startAndWaitProcess(t, "printf \""+strings.Join(outputLines, "\n")+"\"") + + realLogs, err := process.ReadAllLogs(mp.Pid) + if err != nil { + t.Fatal(err) + } + + type TestCase struct { + expectedLogs []*process.LogMessage + queryString string + } + + cases := []TestCase{ + { + expectedLogs: realLogs[5:], + queryString: "limit=5", + }, + { + expectedLogs: realLogs[:5], + queryString: "skip=5", + }, + { + expectedLogs: realLogs[3:5], + queryString: "limit=2&skip=5", + }, + { + expectedLogs: make([]*process.LogMessage, 0), + queryString: "limit=2&skip=20", + }, + { + expectedLogs: realLogs[9:], + queryString: "limit=1", + }, + { + expectedLogs: realLogs[6:], + queryString: query("from", realLogs[6].Time.Format(process.DateTimeFormat)), + }, + { + expectedLogs: realLogs[6:8], + queryString: query( + "from", realLogs[6].Time.Format(process.DateTimeFormat), + "till", realLogs[7].Time.Format(process.DateTimeFormat), + ), + }, + } + + strPid := strconv.Itoa(int(mp.Pid)) + baseURL := "/process/" + strconv.Itoa(int(mp.Pid)) + "/logs?" + + for _, theCase := range cases { + // fetch logs + req, err := http.NewRequest("GET", baseURL+theCase.queryString, nil) + if err != nil { + t.Fatal(err) + } + rr := httptest.NewRecorder() + asHTTPHandlerFunc(getProcessLogsHF, "pid", strPid).ServeHTTP(rr, req) + + // must be 200ok + failIfDifferent(t, http.StatusOK, rr.Code, "status code") + + // check logs are the same to expected + logs := []*process.LogMessage{} + json.Unmarshal(rr.Body.Bytes(), &logs) + failIfDifferent(t, len(theCase.expectedLogs), len(logs), "logs len") + for i := 0; i < len(theCase.expectedLogs); i++ { + failIfDifferent(t, *theCase.expectedLogs[i], *logs[i], "log messages") + } + } +} + +func query(kv ...string) string { + if len(kv) == 0 { + return "" + } + values := url.Values{} + for i := 0; i < len(kv); i += 2 { + values.Add(kv[i], kv[i+1]) + } + return values.Encode() +} + +func asJSONReader(t *testing.T, v interface{}) *bytes.Reader { + body, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return bytes.NewReader(body) +} + +func startAndWaitProcess(t *testing.T, cmd string) process.MachineProcess { + captor := processtest.NewEventsCaptor(process.DiedEventType) + captor.Capture() + + pb := process.NewBuilder() + pb.CmdLine(cmd) + pb.SubscribeDefault("test", captor) + + mp, err := pb.Start() + if err != nil { + captor.Stop() + t.Fatal(err) + } + + if ok := <-captor.Wait(2 * time.Second); !ok { + t.Errorf("Waited 2 seconds for process to finish, killing the process %d", mp.Pid) + if err := process.Kill(mp.Pid); err != nil { + t.Error(err) + } + t.FailNow() + } + + return mp +} + +func failIfDifferent(t *testing.T, expected interface{}, actual interface{}, context string) { + if expected != actual { + t.Fatalf("Expected to receive '%v' %s but received '%v'", expected, context, actual) + } +} + +func failIfFalse(t *testing.T, condition bool, context string) { + if !condition { + t.Fatalf("%s: false", context) + } +} + +func asHTTPHandlerFunc(f rest.HTTPRouteHandlerFunc, params ...string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if err := f(w, r, newFakeParams(params...)); err != nil { + rest.WriteError(w, err) + } + } +} + +func newFakeParams(kv ...string) *fakeParams { + params := &fakeParams{make(map[string]string)} + for i := 0; i < len(kv); i += 2 { + params.items[kv[i]] = kv[i+1] + } + return params +} + +type fakeParams struct { + items map[string]string +} + +func (p *fakeParams) Get(key string) string { + return p.items[key] +}