upport log_max_age

.gitignore

@@ -1 +1,3 @@
-*.test
+*.test
+.DS_Store
+.idea/

.travis.yml

@@ -1,18 +1,9 @@
 language: go
 go:
- - 1.1.2
- - 1.1.1
- - 1.1
  - tip
 
 install:
- - export PATH=$HOME/gopath/bin:$PATH
- - go get -v github.com/axw/gocov
- - go install github.com/axw/gocov/gocov
- - go get -v github.com/golang/glog
- - go get -v launchpad.net/gocheck
- - go get -v github.com/mailgun/glogutils
+ - go get -t .
 
 script:
  - go test -v ./...
- - gocov test -exclude-goroot | gocov report

glogutils.go

@@ -3,20 +3,30 @@ package glogutils
 import (
 	"flag"
 	"fmt"
-	"github.com/golang/glog"
 	"os"
 	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
 )
 
+const DEFAULT_LOGMAXAGE = 0 // default max age in days
+
+var logMaxAge = flag.Int("log_max_age", DEFAULT_LOGMAXAGE,
+	"If non-empty, it determines the maximum # of days (exclusive) to retain old log files.")
+
 // Removes old logs generated by golang, should be called in some separate goroutine
 func CleanupLogs() error {
 	currentDir := LogDir()
 	if currentDir == "" {
 		glog.Infof("Glog's log dir is not set, nothing to clean up")
 		return nil
 	}
-	glog.Infof("Will clean up logs in: %s", currentDir)
-	return removeFiles(currentDir, programName())
+	logMaxAge := LogMaxAge()
+	glog.Infof("Will clean up logs olderthan maxage [%d days] in: %s", logMaxAge, currentDir)
+	return removeFiles(currentDir, programName(), logMaxAge)
 }
 
 // Shortcut to check if the path is actually a dir
@@ -37,18 +47,30 @@ func programName() string {
 // Determines whether golang's log_dir has been set without overriding it
 func LogDir() string {
 	logDir := ""
-	flag.Visit(func(f *flag.Flag) {
-		if f.Name == "log_dir" {
-			logDir = f.Value.String()
-		}
-	})
+	if f := flag.Lookup("log_dir"); f != nil {
+		logDir = f.Value.String()
+	}
+
 	return logDir
 }
 
+// Determines whether log_max_age has been set without overriding it.
+// LogMaxAge is the maximum # of days (exclusive) to retain old files based on the timestamp encoded in their filename.
+// log_max_age <= 0 means delete immediately
+func LogMaxAge() int {
+	if f := flag.Lookup("log_max_age"); f != nil {
+		if v, err := strconv.Atoi(f.Value.String()); err == nil {
+			*logMaxAge = v
+		}
+	}
+	return *logMaxAge
+}
+
 // Function that removes files in the logDir matching prefix
 // It does not touch symlinks and the files that are being referenced
 // by symlinks, it also skips directories
-func removeFiles(logDir string, prefix string) error {
+// If logMaxAge > 0, it retains the files that are created within logMaxAge days.
+func removeFiles(logDir string, prefix string, maxAge int) error {
 	pattern := fmt.Sprintf("%s/%s*", logDir, prefix)
 
 	files, err := filepath.Glob(pattern)
@@ -60,6 +82,8 @@ func removeFiles(logDir string, prefix string) error {
 	}
 
 	toSkip := make(map[string]bool)
+	timeToRetain := time.Now().Add(-time.Duration(int64(24*time.Hour) * int64(maxAge)))
+
 	for _, file := range files {
 		dir, err := isDir(file)
 		if err != nil {
@@ -73,9 +97,19 @@ func removeFiles(logDir string, prefix string) error {
 				glog.Errorf("Failed to eval symlink %s", file, err)
 				return err
 			}
+
 			if realFile != file {
 				toSkip[file] = true
 				toSkip[realFile] = true
+			} else if maxAge > 0 {
+				// extract the date from filename ('%s.%s.%s.log.%s.%04d%02d%02d-%02d%02d%02d.%d')
+				// format refers to logName func in github.com/golang/glog/blob/master/glog_file.go
+				if logName := strings.Split(realFile, "."); len(logName) > 2 /* avoid panic */ {
+					d, err := time.ParseInLocation("20060102", strings.SplitN(logName[len(logName)-2], "-", 2)[0], time.Local)
+					if err == nil && d.After(timeToRetain) {
+						toSkip[realFile] = true
+					}
+				}
 			}
 		}
 	}

glogutils_test.go

@@ -4,10 +4,12 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
-	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
+
+	. "launchpad.net/gocheck"
 )
 
 func Test(t *testing.T) { TestingT(t) }
@@ -19,7 +21,10 @@ type LogUtilsSuite struct {
 var _ = Suite(&LogUtilsSuite{})
 
 func (s *LogUtilsSuite) SetUpTest(c *C) {
-	tempDir, err := ioutil.TempDir(os.TempDir(), "vulcan_test")
+	// /tmp a symlink to /private/tmp on Mac OS X
+	// so use local directory to avoid skipping all files
+	dir, _ := filepath.Abs(filepath.Dir("."))
+	tempDir, err := ioutil.TempDir(dir, "vulcan_test")
 	if err != nil {
 		panic(err)
 	}
@@ -89,7 +94,7 @@ func (s *LogUtilsSuite) TestRemoveFiles(c *C) {
 		},
 		[]string{"vulcan.directory", "other.directory"},
 	)
-	removeFiles(s.currentDir, "vulcan")
+	removeFiles(s.currentDir, "vulcan", 0)
 	files, err := filepath.Glob(fmt.Sprintf("%s/*", s.currentDir))
 	if err != nil {
 		panic(err)
@@ -126,3 +131,82 @@ func (s *LogUtilsSuite) TestCleanupLogs(c *C) {
 	flag.Set("log_dir", "/something/that/does/not/exist")
 	c.Assert(CleanupLogs(), IsNil)
 }
+
+func (s *LogUtilsSuite) TestRemoveFilesOlderThanMaxAge(c *C) {
+	t := time.Now() // now
+	today := fmt.Sprintf("%04d%02d%02d-%02d%02d%02d",
+		t.Year(),
+		t.Month(),
+		t.Day(),
+		t.Hour(),
+		t.Minute(),
+		t.Second(),
+	)
+	t = t.Add(-time.Duration(int64(24*time.Hour) * int64(3))) // 3 days ago from now
+	threeDaysAgo := fmt.Sprintf("%04d%02d%02d-%02d%02d%02d",
+		t.Year(),
+		t.Month(),
+		t.Day(),
+		t.Hour(),
+		t.Minute(),
+		t.Second(),
+	)
+
+	s.createFiles(
+		[]string{
+			// very old logs to be removed
+			"vulcan.radar1.mg.log.INFO.20131004-221312.25058",
+			"vulcan.radar1.mg.log.ERROR.20131005-005231.31443",
+			"vulcan.radar1.mg.log.WARNING.20131005-005231.31443",
+
+			// 3 days ago logs to be retained
+			fmt.Sprintf("vulcan.radar1.mg.log.INFO.%s.12345", threeDaysAgo),
+			fmt.Sprintf("vulcan.radar1.mg.log.WARNING.%s.12345", threeDaysAgo),
+			fmt.Sprintf("vulcan.radar1.mg.log.ERROR.%s.12345", threeDaysAgo),
+
+			// active logs referenced by symlinks
+			fmt.Sprintf("vulcan.radar1.mg.log.INFO.%s.365", today),
+			fmt.Sprintf("vulcan.radar1.mg.log.WARNING.%s.365", today),
+			fmt.Sprintf("vulcan.radar1.mg.log.ERROR.%s.365", today),
+
+			// totally unrellated logs
+			"mgcore-0.log",
+			"mongo-radar.log.2012-12-07T05-26-56",
+			"redis-test.log",
+		},
+		map[string]string{
+			"vulcan.INFO":  fmt.Sprintf("vulcan.radar1.mg.log.INFO.%s.365", today),
+			"vulcan.WARN":  fmt.Sprintf("vulcan.radar1.mg.log.WARNING.%s.365", today),
+			"vulcan.ERROR": fmt.Sprintf("vulcan.radar1.mg.log.ERROR.%s.365", today),
+		},
+		[]string{"vulcan.directory", "other.directory"},
+	)
+	removeFiles(s.currentDir, "vulcan", 4) // retain logs within 4 days
+	files, err := filepath.Glob(fmt.Sprintf("%s/*", s.currentDir))
+	if err != nil {
+		panic(err)
+	}
+	filesMap := make(map[string]bool, len(files))
+	for _, path := range files {
+		_, fileName := filepath.Split(path)
+		filesMap[fileName] = true
+	}
+	expected := map[string]bool{
+		fmt.Sprintf("vulcan.radar1.mg.log.INFO.%s.12345", threeDaysAgo):    true,
+		fmt.Sprintf("vulcan.radar1.mg.log.WARNING.%s.12345", threeDaysAgo): true,
+		fmt.Sprintf("vulcan.radar1.mg.log.ERROR.%s.12345", threeDaysAgo):   true,
+		fmt.Sprintf("vulcan.radar1.mg.log.INFO.%s.365", today):             true,
+		fmt.Sprintf("vulcan.radar1.mg.log.WARNING.%s.365", today):          true,
+		fmt.Sprintf("vulcan.radar1.mg.log.ERROR.%s.365", today):            true,
+		"vulcan.INFO":      true,
+		"vulcan.WARN":      true,
+		"vulcan.ERROR":     true,
+		"vulcan.directory": true,
+		"other.directory":  true,
+	}
+	for fileName, _ := range expected {
+		if filesMap[fileName] != true {
+			c.Errorf("Expected %s to be present", fileName)
+		}
+	}
+}