feat(api): add API endpoint to retrieve logs

README.md

@@ -52,7 +52,7 @@ You can configure the daemon by:
 | `APP_SCRIPTS_DIR` | `./scripts` | Scripts directory |
 | `APP_SCRIPTS_GIT_URL` | none | GIT repository that contains scripts (Note: this is only used by the Docker image or by using the Docker entrypoint script) |
 | `APP_SCRIPTS_GIT_KEY` | none | GIT SSH private key used to clone the repository (Note: this is only used by the Docker image or by using the Docker entrypoint script) |
-| `APP_WORKING_DIR` | `/tmp` (OS temp dir) | Working directory (to store execution logs) |
+| `APP_LOG_DIR` | `/tmp` (OS temp dir) | Directory to store execution logs |
 | `APP_NOTIFIER` | none | Post script notification (`http` or `smtp`) |
 | `APP_NOTIFIER_FROM` | none | Sender of the notification |
 | `APP_NOTIFIER_TO` | none | Recipient of the notification |
@@ -110,7 +110,7 @@ echo "bar bar bar"
 ```
 
 ```bash
-$ curl -XPOST http://localhost/foo/bar
+$ curl -XPOST http://localhost:8080/foo/bar
 data: foo foo foo
 
 data: bar bar bar
@@ -147,7 +147,7 @@ echo "Script parameters: $1"
 The result:
 
 ```bash
-$ curl --data @test.json http://localhost/echo?foo=bar
+$ curl --data @test.json http://localhost:8080/echo?foo=bar
 data: Query parameter: foo=bar
 
 data: Header parameter: user-agent=curl/7.52.1
@@ -169,7 +169,28 @@ You can override this global behavior per request by setting the HTTP header:
 *Example:*
 
 ```bash
-$ curl -XPOST -H "X-Hook-Timeout: 5" http://localhost/echo?foo=bar
+$ curl -XPOST -H "X-Hook-Timeout: 5" http://localhost:8080/echo?foo=bar
+```
+
+### Webhook logs
+
+As mentioned above, web hook logs are stream in real time during the call.
+However, you can retrieve the logs of a previous call by using the hook ID: `http://localhost:8080/<NAME>/<ID>`
+
+The hook ID is returned as an HTTP header with the Webhook response: `X-Hook-ID`
+
+*Example:*
+
+```bash
+$ # Call webhook
+$ curl -v -XPOST http://localhost:8080/echo?foo=bar
+...
+< HTTP/1.1 200 OK
+< Content-Type: text/event-stream
+< X-Hook-Id: 2
+...
+$ # Retrieve logs afterwards
+$ curl http://localhost:8080/echo/2
 ```
 
 ### Post hook notifications

pkg/api/index.go

@@ -2,8 +2,10 @@ package api
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
 
@@ -33,17 +35,24 @@ func index(conf *config.Config) http.Handler {
 }
 
 func webhookHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method == "POST" {
+		triggerWebhook(w, r)
+	} else if r.Method == "GET" {
+		getWebhookLog(w, r)
+	} else {
+		http.Error(w, "405 Method Not Allowed", http.StatusMethodNotAllowed)
+		return
+	}
+}
+
+func triggerWebhook(w http.ResponseWriter, r *http.Request) {
+	// Check that streaming is supported
 	flusher, ok := w.(http.Flusher)
 	if !ok {
 		http.Error(w, "Streaming not supported!", http.StatusInternalServerError)
 		return
 	}
 
-	if r.Method != "POST" {
-		http.Error(w, "405 Method Not Allowed", http.StatusMethodNotAllowed)
-		return
-	}
-
 	// Get script location
 	p := strings.TrimPrefix(r.URL.Path, "/")
 	script, err := tools.ResolveScript(scriptDir, p)
@@ -76,6 +85,7 @@ func webhookHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Cache-Control", "no-cache")
 	w.Header().Set("Connection", "keep-alive")
 	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("X-Hook-ID", strconv.FormatUint(work.ID, 10))
 
 	for {
 		msg, open := <-work.MessageChan
@@ -90,3 +100,34 @@ func webhookHandler(w http.ResponseWriter, r *http.Request) {
 		flusher.Flush()
 	}
 }
+
+func getWebhookLog(w http.ResponseWriter, r *http.Request) {
+	// Get hook ID
+	id := path.Base(r.URL.Path)
+
+	// Get script location
+	name := path.Dir(strings.TrimPrefix(r.URL.Path, "/"))
+	_, err := tools.ResolveScript(scriptDir, name)
+	if err != nil {
+		logger.Error.Println(err.Error())
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+
+	// Get log file
+	logFile, err := worker.GetLogFile(id, name)
+	if err != nil {
+		logger.Error.Println(err.Error())
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if logFile == nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+	defer logFile.Close()
+
+	w.Header().Set("Content-Type", "text/plain")
+
+	io.Copy(w, logFile)
+}

pkg/worker/work_log.go

@@ -0,0 +1,38 @@
+package worker
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/ncarlier/webhookd/pkg/tools"
+)
+
+// getLogDir returns log directory
+func getLogDir() string {
+	if value, ok := os.LookupEnv("APP_LOG_DIR"); ok {
+		return value
+	}
+	return os.TempDir()
+}
+
+func createLogFile(work *WorkRequest) (*os.File, error) {
+	logFilename := path.Join(getLogDir(), fmt.Sprintf("%s_%d_%s.txt", tools.ToSnakeCase(work.Name), work.ID, time.Now().Format("20060102_1504")))
+	return os.Create(logFilename)
+}
+
+// GetLogFile retrieve work log with its name and id
+func GetLogFile(id, name string) (*os.File, error) {
+	logPattern := path.Join(getLogDir(), fmt.Sprintf("%s_%s_*.txt", tools.ToSnakeCase(name), id))
+	files, err := filepath.Glob(logPattern)
+	if err != nil {
+		return nil, err
+	}
+	if len(files) > 0 {
+		filename := files[len(files)-1]
+		return os.Open(filename)
+	}
+	return nil, nil
+}

pkg/worker/work_runner.go

@@ -2,17 +2,14 @@ package worker
 
 import (
 	"bufio"
-	"fmt"
 	"io"
 	"os"
 	"os/exec"
-	"path"
 	"sync"
 	"syscall"
 	"time"
 
 	"github.com/ncarlier/webhookd/pkg/logger"
-	"github.com/ncarlier/webhookd/pkg/tools"
 )
 
 // ChanWriter is a simple writer to a channel of byte.
@@ -25,22 +22,14 @@ func (c *ChanWriter) Write(p []byte) (int, error) {
 	return len(p), nil
 }
 
-var (
-	workingdir = os.Getenv("APP_WORKING_DIR")
-)
-
-func run(work *WorkRequest) (string, error) {
-	if workingdir == "" {
-		workingdir = os.TempDir()
-	}
-
+func run(work *WorkRequest) error {
 	logger.Info.Printf("Work %s#%d started...\n", work.Name, work.ID)
 	logger.Debug.Printf("Work %s#%d script: %s\n", work.Name, work.ID, work.Script)
 	logger.Debug.Printf("Work %s#%d parameter: %v\n", work.Name, work.ID, work.Args)
 
 	binary, err := exec.LookPath(work.Script)
 	if err != nil {
-		return "", err
+		return err
 	}
 
 	// Exec script with args...
@@ -50,14 +39,13 @@ func run(work *WorkRequest) (string, error) {
 	// using a process group...
 	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 
-	// Open the out file for writing
-	logFilename := path.Join(workingdir, fmt.Sprintf("%s_%d_%s.txt", tools.ToSnakeCase(work.Name), work.ID, time.Now().Format("20060102_1504")))
-	logFile, err := os.Create(logFilename)
+	// Open the log file for writing
+	logFile, err := createLogFile(work)
 	if err != nil {
-		return "", err
+		return err
 	}
 	defer logFile.Close()
-	logger.Debug.Printf("Work %s#%d output to file: %s\n", work.Name, work.ID, logFilename)
+	logger.Debug.Printf("Work %s#%d output to file: %s\n", work.Name, work.ID, logFile.Name())
 
 	wLogFile := bufio.NewWriter(logFile)
 	defer wLogFile.Flush()
@@ -65,18 +53,18 @@ func run(work *WorkRequest) (string, error) {
 	// Combine cmd stdout and stderr
 	outReader, err := cmd.StdoutPipe()
 	if err != nil {
-		return logFilename, err
+		return err
 	}
 	errReader, err := cmd.StderrPipe()
 	if err != nil {
-		return logFilename, err
+		return err
 	}
 	cmdReader := io.MultiReader(outReader, errReader)
 
 	// Start the script...
 	err = cmd.Start()
 	if err != nil {
-		return logFilename, err
+		return err
 	}
 
 	// Create wait group to wait for command output completion
@@ -97,7 +85,7 @@ func run(work *WorkRequest) (string, error) {
 			}
 			// writing to outfile
 			if _, err := wLogFile.WriteString(line + "\n"); err != nil {
-				logger.Error.Println("Error while writing into the log file:", logFilename, err)
+				logger.Error.Println("Error while writing into the log file:", logFile.Name(), err)
 				break
 			}
 		}
@@ -127,8 +115,8 @@ func run(work *WorkRequest) (string, error) {
 
 	if err != nil {
 		logger.Info.Printf("Work %s#%d done [ERROR]\n", work.Name, work.ID)
-		return logFilename, err
+		return err
 	}
 	logger.Info.Printf("Work %s#%d done [SUCCESS]\n", work.Name, work.ID)
-	return logFilename, nil
+	return nil
 }

pkg/worker/work_runner_test.go

@@ -1,6 +1,7 @@
 package worker
 
 import (
+	"strconv"
 	"testing"
 
 	"github.com/ncarlier/webhookd/pkg/assert"
@@ -30,8 +31,15 @@ func TestWorkRunner(t *testing.T) {
 	work := NewWorkRequest("test", script, payload, args, 5)
 	assert.NotNil(t, work, "")
 	printWorkMessages(work)
-	_, err := run(work)
+	err := run(work)
 	assert.Nil(t, err, "")
+
+	// Test that log file is ok
+	id := strconv.FormatUint(work.ID, 10)
+	logFile, err := GetLogFile(id, "test")
+	defer logFile.Close()
+	assert.Nil(t, err, "Log file should exists")
+	assert.NotNil(t, logFile, "Log file should be retrieve")
 }
 
 func TestWorkRunnerWithError(t *testing.T) {
@@ -40,7 +48,7 @@ func TestWorkRunnerWithError(t *testing.T) {
 	work := NewWorkRequest("test", script, "", []string{}, 5)
 	assert.NotNil(t, work, "")
 	printWorkMessages(work)
-	_, err := run(work)
+	err := run(work)
 	assert.NotNil(t, err, "")
 	assert.Equal(t, "exit status 1", err.Error(), "")
 }
@@ -51,7 +59,7 @@ func TestWorkRunnerWithTimeout(t *testing.T) {
 	work := NewWorkRequest("test", script, "", []string{}, 1)
 	assert.NotNil(t, work, "")
 	printWorkMessages(work)
-	_, err := run(work)
+	err := run(work)
 	assert.NotNil(t, err, "")
 	assert.Equal(t, "signal: killed", err.Error(), "")
 }

pkg/worker/worker.go

@@ -42,15 +42,15 @@ func (w Worker) Start() {
 			case work := <-w.Work:
 				// Receive a work request.
 				logger.Debug.Printf("Worker #%d received work request: %s#%d\n", w.ID, work.Name, work.ID)
-				filename, err := run(&work)
+				err := run(&work)
 				if err != nil {
-					subject := fmt.Sprintf("Webhook %s#%d FAILED.", work.Name, work.ID)
+					// subject := fmt.Sprintf("Webhook %s#%d FAILED.", work.Name, work.ID)
 					work.MessageChan <- []byte(fmt.Sprintf("error: %s", err.Error()))
-					notify(subject, err.Error(), filename)
+					// notify(subject, err.Error(), filename)
 				} else {
-					subject := fmt.Sprintf("Webhook %s#%d SUCCEEDED.", work.Name, workID)
+					// subject := fmt.Sprintf("Webhook %s#%d SUCCEEDED.", work.Name, workID)
 					work.MessageChan <- []byte("done")
-					notify(subject, "See attachment.", filename)
+					// notify(subject, "See attachment.", filename)
 				}
 				close(work.MessageChan)
 			case <-w.QuitChan: