Type message logs by msg, level, error and context

Server/logging/analysis/analysis.go

@@ -65,8 +65,11 @@ func (l LogLevel) String() string {
 	}
 }
 
-type EventName string
-type MessageName string
+type MetricsLogEventName string
+type MessageLogKey string
+type MessageLogName string
+type MessageLogContext string
+type MessageLogError string
 type LogFields logrus.Fields
 type node map[string]interface{}
 
@@ -84,13 +87,15 @@ type BaseLogModel struct {
 
 type MessageLogModel struct {
 	BaseLogModel
-	Msg   MessageName
-	Level LogLevel
+	Msg               MessageLogName
+	Level             LogLevel
+	MessageLogContext *MessageLogContext
+	MessageLogError   *MessageLogError
 }
 
 type MetricsLogModel struct {
 	BaseLogModel
-	Event EventName
+	Event MetricsLogEventName
 }
 
 type UnknownLogModel struct {
@@ -101,8 +106,46 @@ func (a *BaseLogModel) equal(b BaseLogModel) bool {
 	return a.Node.equal(b.Node)
 }
 
+func (a *MessageLogModel) key() MessageLogKey {
+	var errorString string
+	var context string
+
+	if a.MessageLogError != nil {
+		errorString = string(*a.MessageLogError)
+	}
+	if a.MessageLogContext != nil {
+		context = string(*a.MessageLogContext)
+	}
+
+	return MessageLogKey(fmt.Sprintf("(%s,%s, %s,%s)", MessageLogKey(a.Msg), MessageLogKey(a.Level), errorString, context))
+}
+
+func (a *MessageLogContext) equal(b *MessageLogContext) bool {
+	if a != nil && b != nil {
+		return *a == *b
+	} else if a == nil && b == nil {
+		return true
+	}
+	return false
+}
+
+func (a *MessageLogError) equal(b *MessageLogError) bool {
+	if a != nil && b != nil {
+		return *a == *b
+	} else if a == nil && b == nil {
+		return true
+	}
+	return false
+}
+
 func (a *MessageLogModel) equal(b MessageLogModel) bool {
-	return (a.Msg == b.Msg) && (a.Level == b.Level)
+	if a.Msg != b.Msg {
+		return false
+	} else if a.Level != b.Level {
+		return false
+	}
+
+	return a.MessageLogContext.equal(b.MessageLogContext) && a.MessageLogError.equal(b.MessageLogError)
 }
 
 func (a *MetricsLogModel) equal(b MetricsLogModel) bool {
@@ -165,7 +208,15 @@ func (a *BaseLogModel) Print(printStructure, printExample bool) {
 
 func (a *MessageLogModel) Print(printStructure, printExample bool) {
 	fmt.Printf("MessageLog\n")
-	fmt.Printf("MessageName: %s\n", a.Msg)
+	fmt.Printf("MessageLogName: %s\n", a.Msg)
+
+	if a.MessageLogError != nil {
+		fmt.Printf("MessageLogError: %s\n", *a.MessageLogError)
+	}
+	if a.MessageLogContext != nil {
+		fmt.Printf("MessageLogContext: %s\n", *a.MessageLogContext)
+	}
+
 	if printStructure {
 		fmt.Printf("Structure: %s\n", a.JsonString())
 	}
@@ -176,7 +227,7 @@ func (a *MessageLogModel) Print(printStructure, printExample bool) {
 
 func (a *MetricsLogModel) Print(printStructure, printExample bool) {
 	fmt.Printf("MetricsLog\n")
-	fmt.Printf("EventName: %s\n", a.Event)
+	fmt.Printf("MetricsLogEventName: %s\n", a.Event)
 	if printStructure {
 		fmt.Printf("Structure: %s\n", a.JsonString())
 	}
@@ -279,12 +330,12 @@ type LogTypeStats struct {
 
 type MessageLogStats struct {
 	LogTypeStats
-	modelStats map[MessageName]*MessageLogModelStats
+	modelStats map[MessageLogKey]*MessageLogModelStats
 }
 
 type MetricsLogStats struct {
 	LogTypeStats
-	modelStats map[EventName]*MetricsLogModelStats
+	modelStats map[MetricsLogEventName]*MetricsLogModelStats
 }
 
 type UnknownLogStats struct {
@@ -363,10 +414,10 @@ type LogStats struct {
 func NewLogStats() (l *LogStats) {
 	l = &LogStats{
 		MessageLogModels: MessageLogStats{
-			modelStats: make(map[MessageName]*MessageLogModelStats),
+			modelStats: make(map[MessageLogKey]*MessageLogModelStats),
 		},
 		MetricsLogModels: MetricsLogStats{
-			modelStats: make(map[EventName]*MetricsLogModelStats),
+			modelStats: make(map[MetricsLogEventName]*MetricsLogModelStats),
 		},
 		UnknownLogModels: UnknownLogStats{
 			modelStats: nil,
@@ -376,14 +427,17 @@ func NewLogStats() (l *LogStats) {
 	return l
 }
 
-func NewLogStatsFromFiles(files []string) (l *LogStats) {
+func NewLogStatsFromFiles(files []string) (l *LogStats, err error) {
 	l = NewLogStats()
 
 	for _, file := range files {
-		l.ParseFile(file)
+		err = l.ParseFile(file)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	return l
+	return l, nil
 }
 
 // ParseFile takes a psiphond log file as input, parses the log lines into log
@@ -424,10 +478,11 @@ func (l *LogStats) ParseLogLine(log string) error {
 	switch v := logModel.(type) {
 	case *MessageLogModel:
 		MessageLogModels.Count += 1
-		if m, ok := MessageLogModels.modelStats[v.Msg]; ok {
+
+		if m, ok := MessageLogModels.modelStats[v.key()]; ok {
 			m.Count += 1
 		} else {
-			MessageLogModels.modelStats[v.Msg] = &MessageLogModelStats{LogModelStats{1}, *v}
+			MessageLogModels.modelStats[v.key()] = &MessageLogModelStats{LogModelStats{1}, *v}
 		}
 	case *MetricsLogModel:
 		l.MetricsLogModels.Count += 1
@@ -473,7 +528,7 @@ func parseLogModel(s string) (LogModel, error) {
 	if m["event_name"] != nil {
 		l = &MetricsLogModel{
 			BaseLogModel: b,
-			Event:        EventName(m["event_name"].(string)),
+			Event:        MetricsLogEventName(m["event_name"].(string)),
 		}
 	} else if m["msg"] != nil && m["level"] != nil {
 		var level LogLevel
@@ -490,10 +545,25 @@ func parseLogModel(s string) (LogModel, error) {
 			return nil, fmt.Errorf("Unexpected log level: %s\n", m["level"].(string))
 		}
 
+		var context *MessageLogContext
+		var err *MessageLogError
+
+		if val, ok := m["context"]; ok {
+			c := MessageLogContext(val.(string))
+			context = &c
+		}
+
+		if val, ok := m["error"]; ok {
+			e := MessageLogError(val.(string))
+			err = &e
+		}
+
 		l = &MessageLogModel{
-			BaseLogModel: b,
-			Msg:          MessageName(m["msg"].(string)),
-			Level:        level,
+			BaseLogModel:      b,
+			Msg:               MessageLogName(m["msg"].(string)),
+			Level:             level,
+			MessageLogContext: context,
+			MessageLogError:   err,
 		}
 	} else {
 		l = &UnknownLogModel{

Server/logging/analysis/analysis_test.go

@@ -25,52 +25,11 @@ import (
 )
 
 func TestLogLinesWithExpectations(t *testing.T) {
-	l := NewLogStats()
+	var l *LogStats
 
-	t.Run("test log model parsing", func(t *testing.T) {
-		for _, expectation := range logLinesWithExpectations() {
-			snapshot := *l
+	t.Run("test log model parsing and sorting", func(t *testing.T) {
+		l = parseLogsAndTestExpectations(logLinesWithExpectations(), t)
 
-			err := l.ParseLogLine(expectation.log)
-
-			// Check that the expectation is valid
-			if !(expectation.expects.error || expectation.expects.message ||
-				expectation.expects.metrics || expectation.expects.unknown) {
-				t.Errorf("Malformed expectation expects nothing")
-				t.FailNow()
-			}
-
-			// Check error expectation
-			if err != nil {
-				if expectation.expects.error != true {
-					t.Errorf("Unexpected error from: %s\n", expectation.log)
-				}
-			}
-
-			// Check message expectation
-			if expectation.expects.message {
-				if l.MessageLogModels.Count != snapshot.MessageLogModels.Count+1 {
-					t.Errorf("Expected message log from: %s\n", expectation.log)
-				}
-			}
-
-			// Check metric expectation
-			if expectation.expects.metrics {
-				if l.MetricsLogModels.Count != snapshot.MetricsLogModels.Count+1 {
-					t.Errorf("Expected metric log model from: %s\n", expectation.log)
-				}
-			}
-
-			// Check unknown expectation
-			if expectation.expects.unknown {
-				if l.UnknownLogModels.Count != snapshot.UnknownLogModels.Count+1 {
-					t.Errorf("Expected unknown log model from: %s\n", expectation.log)
-				}
-			}
-		}
-	})
-
-	t.Run("test log model sorting", func(t *testing.T) {
 		logs, _ := l.SortLogModels(true, true, true)
 		var prevLogCount uint
 
@@ -96,6 +55,36 @@ func TestLogLinesWithExpectations(t *testing.T) {
 	})
 }
 
+func TestMessageLogsWithErrorAndContext(t *testing.T) {
+	logs := []LogLineWithExpectation{
+		// The following messages should parse into 4 distinct log models
+		messageLogExpectation(`{"msg":"a", "level":"info"}`),
+		messageLogExpectation(`{"msg":"a", "level":"info"}`),
+		messageLogExpectation(`{"msg":"a", "level":"info", "error": "b"}`),
+		messageLogExpectation(`{"msg":"a", "level":"info", "error": "b"}`),
+		messageLogExpectation(`{"msg":"a", "level":"info", "context": "c"}`),
+		messageLogExpectation(`{"msg":"a", "level":"info", "context": "c"}`),
+		messageLogExpectation(`{"msg":"a", "level":"info", "error": "b", "context": "c"}`),
+		messageLogExpectation(`{"msg":"a", "level":"info", "error": "b", "context": "c"}`),
+
+		// The following messages should parse into 2 distinct log models
+		messageLogExpectation(`{"msg":"b", "level":"info", "error": "b"}`),
+		messageLogExpectation(`{"msg":"b", "level":"info", "context": "b"}`),
+
+		// The following messages should parse into 2 distinct log models
+		messageLogExpectation(`{"msg":"c", "level":"info"}`),
+		messageLogExpectation(`{"msg":"c", "level":"warning"}`),
+	}
+
+	l := parseLogsAndTestExpectations(logs, t)
+
+	numLogModels := len(l.MessageLogModels.modelStats)
+	expectedUniqueModels := 8
+	if numLogModels != expectedUniqueModels {
+		t.Errorf("Expected %d message log models but found %d\n", expectedUniqueModels, numLogModels)
+	}
+}
+
 // Helpers
 
 type LogLineWithExpectation struct {
@@ -110,6 +99,53 @@ type parseLineExpectation struct {
 	unknown bool
 }
 
+func parseLogsAndTestExpectations(expectations []LogLineWithExpectation, t *testing.T) (l *LogStats) {
+	l = NewLogStats()
+
+	for _, expectation := range expectations {
+		snapshot := *l
+
+		err := l.ParseLogLine(expectation.log)
+
+		// Check that the expectation is valid
+		if !(expectation.expects.error || expectation.expects.message ||
+			expectation.expects.metrics || expectation.expects.unknown) {
+			t.Errorf("Malformed expectation expects nothing")
+			t.FailNow()
+		}
+
+		// Check error expectation
+		if err != nil {
+			if expectation.expects.error != true {
+				t.Errorf("Unexpected error: < %s >, from log line: \"%s\"\n", err, expectation.log)
+			}
+		}
+
+		// Check message expectation
+		if expectation.expects.message {
+			if l.MessageLogModels.Count != snapshot.MessageLogModels.Count+1 {
+				t.Errorf("Expected message log from: \"%s\"\n", expectation.log)
+			}
+		}
+
+		// Check metric expectation
+		if expectation.expects.metrics {
+			if l.MetricsLogModels.Count != snapshot.MetricsLogModels.Count+1 {
+				t.Errorf("Expected metric log model from: \"%s\"\n", expectation.log)
+			}
+		}
+
+		// Check unknown expectation
+		if expectation.expects.unknown {
+			if l.UnknownLogModels.Count != snapshot.UnknownLogModels.Count+1 {
+				t.Errorf("Expected unknown log model from: \"%s\"\n", expectation.log)
+			}
+		}
+	}
+
+	return l
+}
+
 func logLinesWithExpectations() (l []LogLineWithExpectation) {
 	l = []LogLineWithExpectation{
 

Server/logging/main.go

@@ -99,7 +99,12 @@ func main() {
 		os.Exit(1)
 	}
 
-	logFileStats := analysis.NewLogStatsFromFiles(logFileList)
+	logFileStats, err := analysis.NewLogStatsFromFiles(logFileList)
+	if err != nil {
+		fmt.Printf("Error while parsing log files: %s\n", err)
+		os.Exit(1)
+	}
+
 	logFileStats.Print(printMessages, printMetrics, printUnknowns, printStructure, printExample)
 
 	fmt.Printf("Found %d messages, %d metrics and %d unknown logs with a total of %d distinct types of logs\n",