audit log config

docker/compose/fluent.json

@@ -0,0 +1,4 @@
+{
+    "fluent_port": 24224,
+    "fluent_host": "fluent"
+}

docker/compose/local-auditlog-compose.yml

@@ -0,0 +1,20 @@
+version: '2'
+
+services:
+  server:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8333:8333
+      - 9333:9333
+      - 19333:19333
+      - 8084:8080
+      - 18084:18080
+      - 8888:8888
+      - 18888:18888
+    command: "server -ip=server -filer -s3 -s3.auditLogConfig=/etc/seaweedfs/fluent.json -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1"
+    volumes:
+      - ./fluent.json:/etc/seaweedfs/fluent.json
+  fluent:
+    image: fluent/fluentd:v1.14
+    ports:
+      - 24224:24224

weed/command/filer.go

@@ -84,6 +84,7 @@ func init() {
 	filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
 	filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
+	filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
 	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 
 	// start webdav on filer

weed/command/s3.go

@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
 	"net/http"
 	"time"
 
@@ -31,6 +32,7 @@ type S3Options struct {
 	tlsCertificate   *string
 	metricsHttpPort  *int
 	allowEmptyFolder *bool
+	auditLogConfig   *string
 }
 
 func init() {
@@ -39,7 +41,7 @@ func init() {
 	s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
 	s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
 	s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
-	s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
+	s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file")
 	s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
 	s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
 	s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
@@ -193,6 +195,10 @@ func (s3opt *S3Options) startS3Server() bool {
 		glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
 	}
 
+	if len(*s3opt.auditLogConfig) > 0 {
+		s3err.InitAuditLog(*s3opt.auditLogConfig)
+	}
+
 	if *s3opt.tlsPrivateKey != "" {
 		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
 		if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {

weed/command/server.go

@@ -131,6 +131,7 @@ func init() {
 	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
 	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
+	s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
 	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 
 	webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")

weed/s3api/s3api_handlers.go

@@ -28,7 +28,7 @@ func (s3a *S3ApiServer) AdjustedUrl(location *filer_pb.Location) string {
 
 func writeSuccessResponseXML(w http.ResponseWriter, r *http.Request, response interface{}) {
 	s3err.WriteXMLResponse(w, r, http.StatusOK, response)
-	s3err.PostLog(r, s3err.ErrNone)
+	s3err.PostLog(r, http.StatusOK, s3err.ErrNone)
 }
 
 func writeSuccessResponseEmpty(w http.ResponseWriter, r *http.Request) {

weed/s3api/s3api_object_handlers.go

@@ -165,11 +165,13 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
 	destUrl := fmt.Sprintf("http://%s%s/%s%s?recursive=true",
 		s3a.option.Filer.ToHttpAddress(), s3a.option.BucketsPath, bucket, urlPathEscape(object))
 
-	s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) {
+	s3a.proxyToFiler(w, r, destUrl, func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int) {
+		statusCode = http.StatusNoContent
 		for k, v := range proxyResponse.Header {
 			w.Header()[k] = v
 		}
-		w.WriteHeader(http.StatusNoContent)
+		w.WriteHeader(statusCode)
+		return statusCode
 	})
 }
 
@@ -224,14 +226,17 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 
 	var deletedObjects []ObjectIdentifier
 	var deleteErrors []DeleteError
+	var auditLog *s3err.AccessLog
 
 	directoriesWithDeletion := make(map[string]int)
 
+	if s3err.Logger != nil {
+		auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
+	}
 	s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		// delete file entries
 		for _, object := range deleteObjects.Objects {
-
 			lastSeparator := strings.LastIndex(object.ObjectName, "/")
 			parentDirectoryPath, entryName, isDeleteData, isRecursive := "", object.ObjectName, true, false
 			if lastSeparator > 0 && lastSeparator+1 < len(object.ObjectName) {
@@ -254,6 +259,10 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
 					Key:     object.ObjectName,
 				})
 			}
+			if auditLog != nil {
+				auditLog.Key = entryName
+				s3err.PostAccessLog(auditLog)
+			}
 		}
 
 		// purge empty folders, only checking folders with deletions
@@ -306,7 +315,7 @@ var passThroughHeaders = []string{
 	"response-expires",
 }
 
-func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResponse *http.Response, w http.ResponseWriter)) {
+func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, destUrl string, responseFn func(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int)) {
 
 	glog.V(3).Infof("s3 proxying %s to %s", r.Method, destUrl)
 
@@ -360,20 +369,23 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 		}
 	}
 
-	responseFn(resp, w)
-	s3err.PostLog(r, s3err.ErrNone)
+	responseStatusCode := responseFn(resp, w)
+	s3err.PostLog(r, responseStatusCode, s3err.ErrNone)
 }
 
-func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) {
+func passThroughResponse(proxyResponse *http.Response, w http.ResponseWriter) (statusCode int) {
 	for k, v := range proxyResponse.Header {
 		w.Header()[k] = v
 	}
 	if proxyResponse.Header.Get("Content-Range") != "" && proxyResponse.StatusCode == 200 {
 		w.WriteHeader(http.StatusPartialContent)
+		statusCode = http.StatusPartialContent
 	} else {
-		w.WriteHeader(proxyResponse.StatusCode)
+		statusCode = proxyResponse.StatusCode
 	}
+	w.WriteHeader(statusCode)
 	io.Copy(w, proxyResponse.Body)
+	return statusCode
 }
 
 func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader io.Reader) (etag string, code s3err.ErrorCode) {

weed/s3api/s3api_object_handlers_postpolicy.go

@@ -142,7 +142,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
 			Location: w.Header().Get("Location"),
 		}
 		s3err.WriteXMLResponse(w, r, http.StatusCreated, resp)
-		s3err.PostLog(r, s3err.ErrNone)
+		s3err.PostLog(r, http.StatusCreated, s3err.ErrNone)
 	case "200":
 		s3err.WriteEmptyResponse(w, r, http.StatusOK)
 	default:

weed/s3api/s3api_object_tagging_handlers.go

@@ -92,7 +92,7 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R
 	}
 
 	w.WriteHeader(http.StatusOK)
-	s3err.PostLog(r, s3err.ErrNone)
+	s3err.PostLog(r, http.StatusOK, s3err.ErrNone)
 }
 
 // DeleteObjectTaggingHandler Delete object tagging
@@ -118,5 +118,5 @@ func (s3a *S3ApiServer) DeleteObjectTaggingHandler(w http.ResponseWriter, r *htt
 	}
 
 	w.WriteHeader(http.StatusNoContent)
-	s3err.PostLog(r, s3err.ErrNone)
+	s3err.PostLog(r, http.StatusNoContent, s3err.ErrNone)
 }

weed/s3api/s3api_server.go

@@ -38,7 +38,6 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
 	s3ApiServer.registerRouter(router)
 
 	go s3ApiServer.subscribeMetaEvents("s3", filer.IamConfigDirecotry+"/"+filer.IamIdentityFile, time.Now().UnixNano())
-
 	return s3ApiServer, nil
 }