diff --git a/.gitignore b/.gitignore index c56069f..415bcaa 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -*.test \ No newline at end of file +*.test +.DS_Store +.idea/ diff --git a/.travis.yml b/.travis.yml index 1851de5..434c7f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,9 @@ language: go go: - - 1.1.2 - - 1.1.1 - - 1.1 - tip install: - - export PATH=$HOME/gopath/bin:$PATH - - go get -v github.com/axw/gocov - - go install github.com/axw/gocov/gocov - - go get -v github.com/golang/glog - - go get -v launchpad.net/gocheck - - go get -v github.com/mailgun/glogutils + - go get -t . script: - go test -v ./... - - gocov test -exclude-goroot | gocov report diff --git a/glogutils.go b/glogutils.go index e17c754..03d9c19 100644 --- a/glogutils.go +++ b/glogutils.go @@ -3,11 +3,20 @@ package glogutils import ( "flag" "fmt" - "github.com/golang/glog" "os" "path/filepath" + "strconv" + "strings" + "time" + + "github.com/golang/glog" ) +const DEFAULT_LOGMAXAGE = 0 // default max age in days + +var logMaxAge = flag.Int("log_max_age", DEFAULT_LOGMAXAGE, + "If non-empty, it determines the maximum # of days (exclusive) to retain old log files.") + // Removes old logs generated by golang, should be called in some separate goroutine func CleanupLogs() error { currentDir := LogDir() @@ -15,8 +24,9 @@ func CleanupLogs() error { glog.Infof("Glog's log dir is not set, nothing to clean up") return nil } - glog.Infof("Will clean up logs in: %s", currentDir) - return removeFiles(currentDir, programName()) + logMaxAge := LogMaxAge() + glog.Infof("Will clean up logs olderthan maxage [%d days] in: %s", logMaxAge, currentDir) + return removeFiles(currentDir, programName(), logMaxAge) } // Shortcut to check if the path is actually a dir @@ -37,18 +47,30 @@ func programName() string { // Determines whether golang's log_dir has been set without overriding it func LogDir() string { logDir := "" - flag.Visit(func(f *flag.Flag) { - if f.Name == "log_dir" { - logDir = f.Value.String() - } - }) + if f := flag.Lookup("log_dir"); f != nil { + logDir = f.Value.String() + } + return logDir } +// Determines whether log_max_age has been set without overriding it. +// LogMaxAge is the maximum # of days (exclusive) to retain old files based on the timestamp encoded in their filename. +// log_max_age <= 0 means delete immediately +func LogMaxAge() int { + if f := flag.Lookup("log_max_age"); f != nil { + if v, err := strconv.Atoi(f.Value.String()); err == nil { + *logMaxAge = v + } + } + return *logMaxAge +} + // Function that removes files in the logDir matching prefix // It does not touch symlinks and the files that are being referenced // by symlinks, it also skips directories -func removeFiles(logDir string, prefix string) error { +// If logMaxAge > 0, it retains the files that are created within logMaxAge days. +func removeFiles(logDir string, prefix string, maxAge int) error { pattern := fmt.Sprintf("%s/%s*", logDir, prefix) files, err := filepath.Glob(pattern) @@ -60,6 +82,8 @@ func removeFiles(logDir string, prefix string) error { } toSkip := make(map[string]bool) + timeToRetain := time.Now().Add(-time.Duration(int64(24*time.Hour) * int64(maxAge))) + for _, file := range files { dir, err := isDir(file) if err != nil { @@ -73,9 +97,19 @@ func removeFiles(logDir string, prefix string) error { glog.Errorf("Failed to eval symlink %s", file, err) return err } + if realFile != file { toSkip[file] = true toSkip[realFile] = true + } else if maxAge > 0 { + // extract the date from filename ('%s.%s.%s.log.%s.%04d%02d%02d-%02d%02d%02d.%d') + // format refers to logName func in github.com/golang/glog/blob/master/glog_file.go + if logName := strings.Split(realFile, "."); len(logName) > 2 /* avoid panic */ { + d, err := time.ParseInLocation("20060102", strings.SplitN(logName[len(logName)-2], "-", 2)[0], time.Local) + if err == nil && d.After(timeToRetain) { + toSkip[realFile] = true + } + } } } } diff --git a/glogutils_test.go b/glogutils_test.go index 2712aae..fbd6d9c 100644 --- a/glogutils_test.go +++ b/glogutils_test.go @@ -4,10 +4,12 @@ import ( "flag" "fmt" "io/ioutil" - . "launchpad.net/gocheck" "os" "path/filepath" "testing" + "time" + + . "launchpad.net/gocheck" ) func Test(t *testing.T) { TestingT(t) } @@ -19,7 +21,10 @@ type LogUtilsSuite struct { var _ = Suite(&LogUtilsSuite{}) func (s *LogUtilsSuite) SetUpTest(c *C) { - tempDir, err := ioutil.TempDir(os.TempDir(), "vulcan_test") + // /tmp a symlink to /private/tmp on Mac OS X + // so use local directory to avoid skipping all files + dir, _ := filepath.Abs(filepath.Dir(".")) + tempDir, err := ioutil.TempDir(dir, "vulcan_test") if err != nil { panic(err) } @@ -89,7 +94,7 @@ func (s *LogUtilsSuite) TestRemoveFiles(c *C) { }, []string{"vulcan.directory", "other.directory"}, ) - removeFiles(s.currentDir, "vulcan") + removeFiles(s.currentDir, "vulcan", 0) files, err := filepath.Glob(fmt.Sprintf("%s/*", s.currentDir)) if err != nil { panic(err) @@ -126,3 +131,82 @@ func (s *LogUtilsSuite) TestCleanupLogs(c *C) { flag.Set("log_dir", "/something/that/does/not/exist") c.Assert(CleanupLogs(), IsNil) } + +func (s *LogUtilsSuite) TestRemoveFilesOlderThanMaxAge(c *C) { + t := time.Now() // now + today := fmt.Sprintf("%04d%02d%02d-%02d%02d%02d", + t.Year(), + t.Month(), + t.Day(), + t.Hour(), + t.Minute(), + t.Second(), + ) + t = t.Add(-time.Duration(int64(24*time.Hour) * int64(3))) // 3 days ago from now + threeDaysAgo := fmt.Sprintf("%04d%02d%02d-%02d%02d%02d", + t.Year(), + t.Month(), + t.Day(), + t.Hour(), + t.Minute(), + t.Second(), + ) + + s.createFiles( + []string{ + // very old logs to be removed + "vulcan.radar1.mg.log.INFO.20131004-221312.25058", + "vulcan.radar1.mg.log.ERROR.20131005-005231.31443", + "vulcan.radar1.mg.log.WARNING.20131005-005231.31443", + + // 3 days ago logs to be retained + fmt.Sprintf("vulcan.radar1.mg.log.INFO.%s.12345", threeDaysAgo), + fmt.Sprintf("vulcan.radar1.mg.log.WARNING.%s.12345", threeDaysAgo), + fmt.Sprintf("vulcan.radar1.mg.log.ERROR.%s.12345", threeDaysAgo), + + // active logs referenced by symlinks + fmt.Sprintf("vulcan.radar1.mg.log.INFO.%s.365", today), + fmt.Sprintf("vulcan.radar1.mg.log.WARNING.%s.365", today), + fmt.Sprintf("vulcan.radar1.mg.log.ERROR.%s.365", today), + + // totally unrellated logs + "mgcore-0.log", + "mongo-radar.log.2012-12-07T05-26-56", + "redis-test.log", + }, + map[string]string{ + "vulcan.INFO": fmt.Sprintf("vulcan.radar1.mg.log.INFO.%s.365", today), + "vulcan.WARN": fmt.Sprintf("vulcan.radar1.mg.log.WARNING.%s.365", today), + "vulcan.ERROR": fmt.Sprintf("vulcan.radar1.mg.log.ERROR.%s.365", today), + }, + []string{"vulcan.directory", "other.directory"}, + ) + removeFiles(s.currentDir, "vulcan", 4) // retain logs within 4 days + files, err := filepath.Glob(fmt.Sprintf("%s/*", s.currentDir)) + if err != nil { + panic(err) + } + filesMap := make(map[string]bool, len(files)) + for _, path := range files { + _, fileName := filepath.Split(path) + filesMap[fileName] = true + } + expected := map[string]bool{ + fmt.Sprintf("vulcan.radar1.mg.log.INFO.%s.12345", threeDaysAgo): true, + fmt.Sprintf("vulcan.radar1.mg.log.WARNING.%s.12345", threeDaysAgo): true, + fmt.Sprintf("vulcan.radar1.mg.log.ERROR.%s.12345", threeDaysAgo): true, + fmt.Sprintf("vulcan.radar1.mg.log.INFO.%s.365", today): true, + fmt.Sprintf("vulcan.radar1.mg.log.WARNING.%s.365", today): true, + fmt.Sprintf("vulcan.radar1.mg.log.ERROR.%s.365", today): true, + "vulcan.INFO": true, + "vulcan.WARN": true, + "vulcan.ERROR": true, + "vulcan.directory": true, + "other.directory": true, + } + for fileName, _ := range expected { + if filesMap[fileName] != true { + c.Errorf("Expected %s to be present", fileName) + } + } +}