From fed78628749fa993438397bbeb797e19f976e400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20B=C3=BClow?= Date: Fri, 15 Aug 2025 19:16:08 +0200 Subject: [PATCH] detect-it-easy and capa scans --- server/internal/database/file.go | 61 +++++++++++++++++- server/internal/database/models.go | 37 +++++++++++ .../database/queries-file_properties.sql | 14 +++++ server/internal/processing/basic/basic.go | 17 +++++ server/internal/processing/capa/capa.go | 63 +++++++++++++++++++ server/internal/processing/types.go | 3 + 6 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 server/internal/database/models.go create mode 100644 server/internal/processing/capa/capa.go diff --git a/server/internal/database/file.go b/server/internal/database/file.go index 364bfd4..27548bf 100644 --- a/server/internal/database/file.go +++ b/server/internal/database/file.go @@ -3,6 +3,7 @@ package database import ( "context" "encoding/hex" + "encoding/json" "log/slog" "git.jmbit.de/jmb/scanfile/server/internal/sqlc" @@ -77,5 +78,63 @@ func GetAllFiles() ([]sqlc.File, error) { slog.Error("Error in GetAllProperties", "error", err) } return files, err - +} + +func InsertFileDiec(params sqlc.InsertFileDIECParams) error { + query := sqlc.New(pool) + err := query.InsertFileDIEC(context.Background(), params) + if err != nil { + slog.Error("Error from query in InsertFileDiec", "file-uuid", params.FileID.String(), "error", err) + } + return err +} + + +func GetFileDiec(fileID pgtype.UUID) (Diec, error){ + var r Diec + var data DiecData + query := sqlc.New(pool) + rawDiec, err := query.GetFileDIEC(context.Background(), fileID) + if err != nil { + slog.Error("Error from query in GetFileDiec", "file-uuid", fileID.String(), "error", err) + return r, err + } + r.ID = rawDiec.ID + r.FileID = rawDiec.FileID + err = json.Unmarshal(rawDiec.Data, &data) + if err != nil { + slog.Error("Error in GetFileDiec", "file-uuid", fileID.String(), "error", err) + return r, err + } + r.Data = data + + return r, nil +} + +func InsertFileCapa(params sqlc.InsertFileCapaParams) error { + query := sqlc.New(pool) + err := query.InsertFileCapa(context.Background(), params) + if err != nil { + slog.Error("Error from Query in InsertFileCapa", "file-uuid", params.FileID.String(), "error", err) + } + return err +} + +func GetFileCapa(fileID pgtype.UUID) (CapaResult, error) { + var r CapaResult + var data CapaData + query := sqlc.New(pool) + rawResult, err := query.GetFileCapa(context.Background(), fileID) + if err != nil { + slog.Error("Error from Query in GetFileCapa", "file-uuid", fileID.String(), "error", err) + } + r.ID = rawResult.ID + r.FileID = rawResult.FileID + err = json.Unmarshal(rawResult.Data, &data) + if err != nil { + slog.Error("Error in GetFileCapa", "file-uuid", fileID.String(), "error", err) + return r, err + } + + return r, nil } diff --git a/server/internal/database/models.go b/server/internal/database/models.go new file mode 100644 index 0000000..4fd68f8 --- /dev/null +++ b/server/internal/database/models.go @@ -0,0 +1,37 @@ +package database + +import "github.com/jackc/pgx/v5/pgtype" + +type DiecData struct { + Detects []struct { + Filetype string `json:"filetype"` + Parentfilepart string `json:"parentfilepart"` + Values []struct { + Info string `json:"info"` + Name string `json:"name"` + String string `json:"string"` + Type string `json:"type"` + Version string `json:"version"` + } `json:"values"` + } `json:"detects"` +} + +type Diec struct { + ID int64 + FileID pgtype.UUID + Data DiecData +} + +type CapaResult struct { + ID int64 + FileID pgtype.UUID + Data CapaData +} + +type CapaData struct { + Capabilities map[string]string `json:"capabilities"` + Maec map[string]string `json:"maec"` + Objectives map[string]string `json:"objectives"` + Tactics map[string]string `json:"tactics"` +} + diff --git a/server/internal/database/queries-file_properties.sql b/server/internal/database/queries-file_properties.sql index 6f270db..8950589 100644 --- a/server/internal/database/queries-file_properties.sql +++ b/server/internal/database/queries-file_properties.sql @@ -13,3 +13,17 @@ INSERT INTO diec ( file_id, data ) VALUES ($1, $2); +-- name: GetFileDIEC :one +SELECT * FROM diec +WHERE file_id = $1; + +-- name: InsertFileCapa :exec +INSERT INTO capa_results ( + file_id, data +) VALUES ($1, $2); + +-- name: GetFileCapa :one +SELECT * FROM capa_results +WHERE file_id = $1; + + diff --git a/server/internal/processing/basic/basic.go b/server/internal/processing/basic/basic.go index 2681524..1f5c115 100644 --- a/server/internal/processing/basic/basic.go +++ b/server/internal/processing/basic/basic.go @@ -43,6 +43,23 @@ func BasicProcessing(job sqlc.ProcessingJob) error { slog.Error("Error inserting basic file properties into database", "file-uuid", job.FileID.String(), "error", err) return err } + diec, err := DiecScan(job.FileID.String()) + if err != nil { + database.FailProcessingJob(job.ID, err) + slog.Error("Error processing file", "file-uuid", job.FileID.String(), "error", err) + return err + } + slog.Debug("Diec result", "file-uuid", job.FileID.String(), "diec", diec) + err = database.InsertFileDiec(sqlc.InsertFileDIECParams{ + FileID: job.FileID, + Data: diec, + }) + if err != nil { + database.FailProcessingJob(job.ID, err) + slog.Error("Error inserting detect-it-easy results into database", "file-uuid", job.FileID.String(), "error", err) + return err + } + database.FinishProcessingJob(job.ID) return nil } diff --git a/server/internal/processing/capa/capa.go b/server/internal/processing/capa/capa.go new file mode 100644 index 0000000..7631fc8 --- /dev/null +++ b/server/internal/processing/capa/capa.go @@ -0,0 +1,63 @@ +package capa + +import ( + "log/slog" + "net/url" + "net/http" + "fmt" + "io" + + "git.jmbit.de/jmb/scanfile/server/internal/database" + "git.jmbit.de/jmb/scanfile/server/internal/sqlc" + "github.com/jackc/pgx/v5/pgtype" + "github.com/spf13/viper" +) + + +func CapaProcessing(job sqlc.ProcessingJob) error { + database.StartProcessingJob(job.ID) + capaBytes, err := CapaScan(job.FileID) + if err != nil { + database.FailProcessingJob(job.ID, err) + slog.Error("Error processing file", "file-uuid", job.FileID.String(), "error", err) + return err + } + var params sqlc.InsertFileCapaParams + params.FileID = job.FileID + params.Data = capaBytes + err = database.InsertFileCapa(params) + if err != nil { + database.FailProcessingJob(job.ID, err) + slog.Error("Error processing file CAPA data", "file-uuid", job.FileID.String(), "error", err) + return err + } + + database.FinishProcessingJob(job.ID) + return nil +} + +func CapaScan(fileID pgtype.UUID) ([]byte, error) { + var by []byte + slog.Debug("Started Capa scan", "file-uuid", fileID.String()) + capaUrl, err := url.Parse(viper.GetString("processing.capaurl")) + if err != nil { + slog.Error("Error in CapaScan parsing URL for ole service", "file-uuid", fileID.String(), "error", err) + return by, err + } + capaUrl.Path = "/capa/analyze" + capaUrl.RawQuery = fmt.Sprintf("file=%s", fileID.String()) + capaResp, err := http.Get(capaUrl.String()) + slog.Debug("CAPA scan request", "file-uuid", fileID.String(), "url", capaUrl.String(), "status-code", capaResp.StatusCode) + if err != nil { + slog.Error("Error in CapaScan getting capa result from service", "file-uuid", fileID.String(), "error", err) + } + defer capaResp.Body.Close() + body, err := io.ReadAll(capaResp.Body) + if err != nil { + slog.Error("Error in CapaScan parsing body", "file-uuid", fileID.String(), "error", err) + } + + slog.Debug("CapaScan", "file-uuid", fileID.String(), "data", body) + return body, nil +} + diff --git a/server/internal/processing/types.go b/server/internal/processing/types.go index 4675024..cabae3f 100644 --- a/server/internal/processing/types.go +++ b/server/internal/processing/types.go @@ -28,6 +28,9 @@ const TypeOther = "Other" // Yara Scan (can be done for all filetypes) const TypeYARA = "Yara" +// CAPA Scan +const TypeCAPA = "Capa" + var MSOfficeMime = []string{ "application/msword", "application/vnd.ms-excel",