This commit is contained in:
Johannes Bülow 2025-06-12 13:00:13 +02:00
parent 1962bd885b
commit 7a3bc3bd75
Signed by: jmb
GPG key ID: B56971CF7B8F83A6
18 changed files with 169 additions and 178 deletions

View file

@ -18,7 +18,7 @@ func SubmitRestHandler(w http.ResponseWriter, r *http.Request) {
return
}
err := r.ParseMultipartForm(viper.GetInt64("web.maxfilesizemb") * 1024 * 1024)
err := r.ParseMultipartForm(viper.GetInt64("web.maxfilesizemb") * 1024 * 1024)
if err != nil {
slog.Error("Error parsing form in SubmitRestHandler", "error", err)
utils.WriteJSONError(w, err.Error(), http.StatusBadRequest)
@ -40,18 +40,18 @@ func SubmitRestHandler(w http.ResponseWriter, r *http.Request) {
file, err := database.CreateFile(r.Context(), fileHeader.Filename, fileBytes)
if err != nil {
slog.Error("Error saving file in SubmitRestHandler", "error", err)
utils.WriteJSONError(w, err.Error(), http.StatusInternalServerError)
utils.WriteJSONError(w, err.Error(), http.StatusInternalServerError)
return
}
err = processing.Submit(r.Context(), file.ID)
w.Header().Set("Content-Type", "application/json")
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
w.WriteHeader(http.StatusOK)
err = enc.Encode(file)
err = processing.Submit(r.Context(), file.ID)
w.Header().Set("Content-Type", "application/json")
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
w.WriteHeader(http.StatusOK)
err = enc.Encode(file)
if err != nil {
slog.Error("Error creating json encoder in SubmitRestHandler", "error", err)
utils.WriteJSONError(w, err.Error(), http.StatusInternalServerError)
utils.WriteJSONError(w, err.Error(), http.StatusInternalServerError)
return
}

View file

@ -37,17 +37,17 @@ func setDefaults() {
viper.SetDefault("web.key", "/etc/ssl/key/ssl-cert-snakeoil.key")
viper.SetDefault("web.loghttp", true)
viper.SetDefault("web.maxfilesizemb", 100)
// Database
// Database
viper.SetDefault("db.host", "localhost")
viper.SetDefault("db.port", 5432)
viper.SetDefault("db.user", "scanfile")
viper.SetDefault("db.database", "scanfile")
viper.SetDefault("db.password", "CHANGEME")
viper.SetDefault("db.debug", false)
// Others
viper.SetDefault("processing.oleurl", "http://localhost:5000")
// Others
viper.SetDefault("processing.oleurl", "http://localhost:5000")
viper.SetDefault("store.path", "./storage/files/")
viper.SetDefault("debug", false)
viper.SetDefault("debug", false)
}
func SaveConfig() error {

View file

@ -2,8 +2,8 @@ package database
import (
"context"
"encoding/hex"
"log/slog"
"encoding/hex"
"github.com/jackc/pgx/v5"
"golang.org/x/crypto/blake2b"
@ -21,29 +21,29 @@ func CreateFile(ctx context.Context, name string, fileBytes []byte) (sqlc.File,
var err error
bl2hash := blake2b.Sum256(fileBytes)
slog.Debug("calculated Blake2b hash", "file-name", name, "file-bl2", hex.EncodeToString(bl2hash[:]), "file-size", len(fileBytes))
mime, _ := store.GetBytesFileType(fileBytes[:262])
slog.Debug("calculated Blake2b hash", "file-name", name, "file-bl2", hex.EncodeToString(bl2hash[:]), "file-size", len(fileBytes))
mime, _ := store.GetBytesFileType(fileBytes[:262])
file, err = queries.CreateFile(ctx, sqlc.CreateFileParams{
Name: name,
Mimetype: mime,
Size: int64(len(fileBytes)),
Blake2: bl2hash[:],
})
Name: name,
Mimetype: mime,
Size: int64(len(fileBytes)),
Blake2: bl2hash[:],
})
if err == pgx.ErrNoRows {
file, err := queries.GetFileByBlake2(ctx, bl2hash[:])
if err != nil {
slog.Error("Error saving file to database", "error", err, "file-name", name)
return file, err
}
slog.Debug("File already exists", "file-uuid", file.ID.String(), "file-name", file.Name, "file-bl2", hex.EncodeToString(file.Blake2), "file-size", file.Size, "file-mime", file.Mimetype, "file-description", file.Description.String)
file, err := queries.GetFileByBlake2(ctx, bl2hash[:])
if err != nil {
slog.Error("Error saving file to database", "error", err, "file-name", name)
return file, err
}
slog.Debug("File already exists", "file-uuid", file.ID.String(), "file-name", file.Name, "file-bl2", hex.EncodeToString(file.Blake2), "file-size", file.Size, "file-mime", file.Mimetype, "file-description", file.Description.String)
return file, nil
}
if err != nil {
slog.Error("Error saving file to database", "error", err, "file-name", name)
err = nil
} else {
slog.Debug("New file created", "file-uuid", file.ID.String(), "file-name", file.Name, "file-bl2", hex.EncodeToString(file.Blake2), "file-size", file.Size, "file-mime", file.Mimetype, "file-description", file.Description.String)
slog.Debug("New file created", "file-uuid", file.ID.String(), "file-name", file.Name, "file-bl2", hex.EncodeToString(file.Blake2), "file-size", file.Size, "file-mime", file.Mimetype, "file-description", file.Description.String)
}
//Using UUIDs instead of the file hash to make switching storage backends easier

View file

@ -24,10 +24,10 @@ func GetFileByID(fileID string) (sqlc.File, error) {
func InsertFileProperties(properties sqlc.InsertFilePropertiesParams) error {
query := sqlc.New(pool)
slog.Debug("InsertFileProperties", "file-uuid", properties.ID.String(), "file-sha256",
hex.EncodeToString(properties.Sha256), "file-md5", hex.EncodeToString(properties.Md5),
"file-mime", properties.LibmagicMime.String, "file-extension", properties.LibmagicExtension.String,
"file-apple", properties.LibmagicApple.String)
slog.Debug("InsertFileProperties", "file-uuid", properties.ID.String(), "file-sha256",
hex.EncodeToString(properties.Sha256), "file-md5", hex.EncodeToString(properties.Md5),
"file-mime", properties.LibmagicMime.String, "file-extension", properties.LibmagicExtension.String,
"file-apple", properties.LibmagicApple.String)
err := query.InsertFileProperties(context.Background(), properties)
if err != nil {
slog.Error("Unable to add file properties", "file-uuid", properties.ID.String(), "error", err)
@ -49,24 +49,24 @@ func InsertJsonResult(fileID pgtype.UUID, data []byte, table string) error {
err = query.InsertFileMsofficeOlevba(context.Background(), sqlc.InsertFileMsofficeOlevbaParams{FileID: fileID, Data: data})
case "msoffice_mraptor":
err = query.InsertFileMsofficeMraptor(context.Background(), sqlc.InsertFileMsofficeMraptorParams{FileID: fileID, Data: data})
default:
err = fmt.Errorf("Invalid table name")
default:
err = fmt.Errorf("Invalid table name")
}
if err != nil {
slog.Error("Unable to insert DIEC results", "file-uuid", fileID.String(), "error", err)
return err
return err
}
slog.Debug("InsertJsonResult", "file-uuid", fileID.String(), "table", table)
slog.Debug("InsertJsonResult", "file-uuid", fileID.String(), "table", table)
return nil
}
// GetFileMime() returns the MimeType for a file
func GetFileMime(fileID pgtype.UUID) (string, error) {
query := sqlc.New(pool)
mimeType, err := query.GetFileMime(context.Background(), fileID)
if err != nil {
slog.Error("Error getting file Mimetype", "file-uuid", fileID.String(), "error", err)
return "", err
}
return mimeType, nil
query := sqlc.New(pool)
mimeType, err := query.GetFileMime(context.Background(), fileID)
if err != nil {
slog.Error("Error getting file Mimetype", "file-uuid", fileID.String(), "error", err)
return "", err
}
return mimeType, nil
}

View file

@ -12,7 +12,7 @@ import (
// BasicProcessing() determines type agnostic information about the file
func BasicProcessing(job sqlc.ProcessingJob) error {
database.StartProcessingJob(job.ID)
database.StartProcessingJob(job.ID)
fileBytes, err := store.GetFileBytes(job.FileID.String())
if err != nil {
database.FailProcessingJob(job.ID, err)

View file

@ -24,8 +24,8 @@ func DiecScan(fileName string) ([]byte, error) {
return by, err
}
if json.Valid(result) == false {
return by, fmt.Errorf("JSON not valid")
}
if json.Valid(result) == false {
return by, fmt.Errorf("JSON not valid")
}
return result, nil
}

View file

@ -12,31 +12,30 @@ import (
"github.com/spf13/viper"
)
// MraptorScan() requests a scan of the file from the ole service
func MraptorScan(fileID pgtype.UUID) error {
slog.Debug("Starting MacroRaptor scan", "file-uuid", fileID.String())
oleidUrl, err := url.Parse(viper.GetString("processing.oleurl"))
if err != nil {
slog.Error("Error parsing URL for ole service", "file-uuid", fileID.String(), "error", err)
}
oleidUrl.Path = "/olevba/analyze"
oleidUrl.Query().Add("file", fileID.String())
oleidResp, err := http.Get(oleidUrl.String())
slog.Debug("MraptorScan request", "file-uuid", fileID.String(), "url", oleidUrl.String(), "status-code", oleidResp.StatusCode)
if err != nil {
slog.Error("Error getting mraptor info from service", "file-uuid", fileID.String(), "error", err)
}
var body []byte
_, err = oleidResp.Body.Read(body)
if err != nil {
slog.Error("Error parsing mraptor body", "file-uuid", fileID.String(), "error", err)
}
slog.Debug("Starting MacroRaptor scan", "file-uuid", fileID.String())
oleidUrl, err := url.Parse(viper.GetString("processing.oleurl"))
if err != nil {
slog.Error("Error parsing URL for ole service", "file-uuid", fileID.String(), "error", err)
}
oleidUrl.Path = "/olevba/analyze"
oleidUrl.Query().Add("file", fileID.String())
oleidResp, err := http.Get(oleidUrl.String())
slog.Debug("MraptorScan request", "file-uuid", fileID.String(), "url", oleidUrl.String(), "status-code", oleidResp.StatusCode)
if err != nil {
slog.Error("Error getting mraptor info from service", "file-uuid", fileID.String(), "error", err)
}
var body []byte
_, err = oleidResp.Body.Read(body)
if err != nil {
slog.Error("Error parsing mraptor body", "file-uuid", fileID.String(), "error", err)
}
if json.Valid(body) == false {
return fmt.Errorf("JSON not valid")
}
slog.Debug("MraptorScan", "file-uuid", fileID.String(), "data", body)
database.InsertJsonResult(fileID, body, "msoffice_mraptor")
return nil
if json.Valid(body) == false {
return fmt.Errorf("JSON not valid")
}
slog.Debug("MraptorScan", "file-uuid", fileID.String(), "data", body)
database.InsertJsonResult(fileID, body, "msoffice_mraptor")
return nil
}

View file

@ -5,23 +5,22 @@ import (
"git.jmbit.de/jmb/scanfile/server/internal/sqlc"
)
func MSOfficeProcessing(job sqlc.ProcessingJob) error {
database.StartProcessingJob(job.ID)
err := OleIDScan(job.FileID)
if err != nil {
database.FailProcessingJob(job.ID, err)
return err
}
err = OleVBAScan(job.FileID)
if err != nil {
database.FailProcessingJob(job.ID, err)
return err
}
err = MraptorScan(job.FileID)
if err != nil {
database.FailProcessingJob(job.ID, err)
return err
}
return nil
database.StartProcessingJob(job.ID)
err := OleIDScan(job.FileID)
if err != nil {
database.FailProcessingJob(job.ID, err)
return err
}
err = OleVBAScan(job.FileID)
if err != nil {
database.FailProcessingJob(job.ID, err)
return err
}
err = MraptorScan(job.FileID)
if err != nil {
database.FailProcessingJob(job.ID, err)
return err
}
return nil
}

View file

@ -12,31 +12,30 @@ import (
"github.com/spf13/viper"
)
func OleIDScan(fileID pgtype.UUID) error {
slog.Debug("Starting OleID scan", "file-uuid", fileID.String())
oleidUrl, err := url.Parse(viper.GetString("processing.oleurl"))
if err != nil {
slog.Error("Error parsing URL for ole service", "file-uuid", fileID.String(), "error", err)
}
oleidUrl.Path = "/oleid/analyze"
oleidUrl.Query().Add("file", fileID.String())
oleidResp, err := http.Get(oleidUrl.String())
slog.Debug("OleIDScan request", "file-uuid", fileID.String(), "url", oleidUrl.String(), "status-code", oleidResp.StatusCode)
if err != nil {
slog.Error("Error getting oleid info from service", "file-uuid", fileID.String(), "error", err)
}
var body []byte
_, err = oleidResp.Body.Read(body)
if err != nil {
slog.Error("Error parsing oleid body", "file-uuid", fileID.String(), "error", err)
}
slog.Debug("Starting OleID scan", "file-uuid", fileID.String())
oleidUrl, err := url.Parse(viper.GetString("processing.oleurl"))
if err != nil {
slog.Error("Error parsing URL for ole service", "file-uuid", fileID.String(), "error", err)
}
oleidUrl.Path = "/oleid/analyze"
oleidUrl.Query().Add("file", fileID.String())
oleidResp, err := http.Get(oleidUrl.String())
slog.Debug("OleIDScan request", "file-uuid", fileID.String(), "url", oleidUrl.String(), "status-code", oleidResp.StatusCode)
if err != nil {
slog.Error("Error getting oleid info from service", "file-uuid", fileID.String(), "error", err)
}
var body []byte
_, err = oleidResp.Body.Read(body)
if err != nil {
slog.Error("Error parsing oleid body", "file-uuid", fileID.String(), "error", err)
}
if json.Valid(body) == false {
return fmt.Errorf("JSON not valid")
}
slog.Debug("OleIDScan", "file-uuid", fileID.String(), "data", body)
database.InsertJsonResult(fileID, body, "msoffice_oleid")
return nil
if json.Valid(body) == false {
return fmt.Errorf("JSON not valid")
}
slog.Debug("OleIDScan", "file-uuid", fileID.String(), "data", body)
database.InsertJsonResult(fileID, body, "msoffice_oleid")
return nil
}

View file

@ -12,30 +12,29 @@ import (
"github.com/spf13/viper"
)
func OleVBAScan(fileID pgtype.UUID) error {
slog.Debug("Starting OLEvba scan", "file-uuid", fileID.String())
oleidUrl, err := url.Parse(viper.GetString("processing.oleurl"))
if err != nil {
slog.Error("Error parsing URL for ole service", "file-uuid", fileID.String(), "error", err)
}
oleidUrl.Path = "/olevba/analyze"
oleidUrl.Query().Add("file", fileID.String())
oleidResp, err := http.Get(oleidUrl.String())
slog.Debug("OleVBAScan request", "file-uuid", fileID.String(), "url", oleidUrl.String(), "status-code", oleidResp.StatusCode)
if err != nil {
slog.Error("Error getting olevba info from service", "file-uuid", fileID.String(), "error", err)
}
var body []byte
_, err = oleidResp.Body.Read(body)
if err != nil {
slog.Error("Error parsing olevba body", "file-uuid", fileID.String(), "error", err)
}
slog.Debug("Starting OLEvba scan", "file-uuid", fileID.String())
oleidUrl, err := url.Parse(viper.GetString("processing.oleurl"))
if err != nil {
slog.Error("Error parsing URL for ole service", "file-uuid", fileID.String(), "error", err)
}
oleidUrl.Path = "/olevba/analyze"
oleidUrl.Query().Add("file", fileID.String())
oleidResp, err := http.Get(oleidUrl.String())
slog.Debug("OleVBAScan request", "file-uuid", fileID.String(), "url", oleidUrl.String(), "status-code", oleidResp.StatusCode)
if err != nil {
slog.Error("Error getting olevba info from service", "file-uuid", fileID.String(), "error", err)
}
var body []byte
_, err = oleidResp.Body.Read(body)
if err != nil {
slog.Error("Error parsing olevba body", "file-uuid", fileID.String(), "error", err)
}
if json.Valid(body) == false {
return fmt.Errorf("JSON not valid")
}
slog.Debug("OleVBAScan", "file-uuid", fileID.String(), "data", body)
database.InsertJsonResult(fileID, body, "msoffice_olevba")
return nil
if json.Valid(body) == false {
return fmt.Errorf("JSON not valid")
}
slog.Debug("OleVBAScan", "file-uuid", fileID.String(), "data", body)
database.InsertJsonResult(fileID, body, "msoffice_olevba")
return nil
}

View file

@ -21,28 +21,27 @@ func Setup(wg *sync.WaitGroup) {
// Submit() starts the analysis process for a file.
func Submit(ctx context.Context, file pgtype.UUID) error {
// Always start a basic task
// Always start a basic task
job, err := database.NewProcessingJob(ctx, file, TypeBasic)
if err != nil {
slog.Error("Could not submit processing job", "error", err, "file-uuid", file, "type", TypeBasic)
return err
}
go basic.BasicProcessing(job)
mimeType, err := database.GetFileMime(file)
go basic.BasicProcessing(job)
mimeType, err := database.GetFileMime(file)
if err != nil {
slog.Error("Could not retrieve MimeType", "error", err, "file-uuid", file)
return err
}
switch TypeFromMime(mimeType) {
case TypeMSOffice:
officeJob, err := database.NewProcessingJob(ctx, file, TypeMSOffice)
if err != nil {
slog.Error("Could not submit processing job", "error", err, "file-uuid", file, "type", TypeMSOffice)
return err
}
go msoffice.MSOfficeProcessing(officeJob)
}
switch TypeFromMime(mimeType) {
case TypeMSOffice:
officeJob, err := database.NewProcessingJob(ctx, file, TypeMSOffice)
if err != nil {
slog.Error("Could not submit processing job", "error", err, "file-uuid", file, "type", TypeMSOffice)
return err
}
go msoffice.MSOfficeProcessing(officeJob)
}
return nil
}

View file

@ -78,4 +78,3 @@ func TypeFromMime(mimetype string) string {
return TypeOther
}

View file

@ -9,16 +9,15 @@ import (
func RegisterRoutes() *http.ServeMux {
mux := http.NewServeMux()
// Web interface
// Web interface
mux.HandleFunc("/", web.IndexWebHandler)
mux.HandleFunc("/about", web.AboutWebHandler)
mux.HandleFunc("/files/{uuid}", web.FileViewWebHandler)
mux.HandleFunc("POST /upload", web.IndexUploadHandler)
mux.Handle("/assets/", http.FileServer(http.FS(web.Files)))
// REST API
mux.HandleFunc("/api/v0/submit", api.SubmitRestHandler)
// REST API
mux.HandleFunc("/api/v0/submit", api.SubmitRestHandler)
return mux
}

View file

@ -21,14 +21,14 @@ func GetFileType(fileId string) (string, error) {
// We only have to pass the file header = first 261 bytes
head := make([]byte, 261)
file.Read(head)
return GetBytesFileType(head)
return GetBytesFileType(head)
}
// Returns the MimeType for a []byte
// We only have to pass the file header = first 261 bytes
func GetBytesFileType(data []byte) (string, error) {
kind, err := filetype.Match(data)
slog.Debug("GetBytesFileType", "data", data, "file-mime", kind.MIME.Value)
slog.Debug("GetBytesFileType", "data", data, "file-mime", kind.MIME.Value)
if err != nil {
slog.Error("Could not determine file type", "error", err)
return "application/octet-stream", err

View file

@ -7,20 +7,18 @@ import (
)
type ErrorResponse struct {
Message string `json:"message"`
Code int `json:"code,omitempty"`
Message string `json:"message"`
Code int `json:"code,omitempty"`
}
func WriteJSONError(w http.ResponseWriter, message string, statusCode int) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(statusCode)
errorRes := ErrorResponse{Message: message, Code: statusCode}
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
err := enc.Encode(errorRes) // Encode and write the JSON response
if err != nil {
slog.Error("Error in WriteJSONError", "error", err)
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(statusCode)
errorRes := ErrorResponse{Message: message, Code: statusCode}
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
err := enc.Encode(errorRes) // Encode and write the JSON response
if err != nil {
slog.Error("Error in WriteJSONError", "error", err)
}
}

View file

@ -34,10 +34,10 @@ func main() {
log.SetOutput(os.Stderr)
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, nil)))
config.ReadConfigFile("")
if viper.GetBool("debug") {
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})))
slog.Debug("Debug logging enabled")
}
if viper.GetBool("debug") {
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})))
slog.Debug("Debug logging enabled")
}
var wg sync.WaitGroup
database.Connect()
database.Ping()

View file

@ -31,14 +31,14 @@ func IndexUploadHandler(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxUploadSize)
defer r.Body.Close()
fileData, fileHeader, err := r.FormFile("file")
slog.Debug("File form submitted", "file-name", fileHeader.Filename, "file-size", fileHeader.Size)
slog.Debug("File form submitted", "file-name", fileHeader.Filename, "file-size", fileHeader.Size)
if err != nil {
slog.Error("Error parsing form in IndexUploadHandler", "error", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
fileBytes, err := io.ReadAll(fileData)
slog.Debug("File from form read", "file-name", fileHeader.Filename, "file-size", len(fileBytes))
slog.Debug("File from form read", "file-name", fileHeader.Filename, "file-size", len(fileBytes))
if err != nil {
slog.Error("Error reading file in IndexUploadHandler", "error", err)
http.Error(w, err.Error(), http.StatusBadRequest)
@ -51,7 +51,7 @@ func IndexUploadHandler(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
err = processing.Submit(r.Context(), file.ID)
err = processing.Submit(r.Context(), file.ID)
if err != nil {
slog.Error("Error submitting file for processing in IndexUploadHandler", "error", err)
http.Error(w, err.Error(), http.StatusInternalServerError)

View file

@ -1 +1 @@
exit status 2exit status 2exit status 2exit status 2exit status 2exit status 2exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1
exit status 2exit status 2exit status 2exit status 2exit status 2exit status 2exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1exit status 1