diff --git a/scanners/ole/routes/oleid.py b/scanners/ole/routes/oleid.py index 4f44f6c..b38567f 100644 --- a/scanners/ole/routes/oleid.py +++ b/scanners/ole/routes/oleid.py @@ -1,5 +1,6 @@ from flask import Blueprint, request, jsonify, abort from os import path +from werkzeug.utils import secure_filename import oletools.oleid import config @@ -7,7 +8,7 @@ oleid_bp = Blueprint('oleid', __name__) @oleid_bp.route('/analyze', methods=['GET']) def analyze_ole(): - file = request.args.get('file', '') + file = secure_filename(request.args.get('file', '')) if file == '': abort(400) filepath = path.join(config.Config.FILE_DIRECTORY, file) @@ -16,4 +17,4 @@ def analyze_ole(): indicators = oid.check() results = {indicator.name: indicator.value for indicator in indicators} - return jsonify({'filename': file, 'result': results}) + return jsonify(results) diff --git a/scanners/ole/routes/olevba.py b/scanners/ole/routes/olevba.py index f49aefc..6d0e87d 100644 --- a/scanners/ole/routes/olevba.py +++ b/scanners/ole/routes/olevba.py @@ -1,5 +1,6 @@ from os import path from flask import Blueprint, request, jsonify, abort +from werkzeug.utils import secure_filename import config import oletools.olevba @@ -7,13 +8,30 @@ olevba_bp = Blueprint('olevba', __name__) @olevba_bp.route('/analyze', methods=['GET']) def analyze_vba(): - file = request.args.get('file', '') + file = secure_filename(request.args.get('file', '')) if file == '': abort(400) filepath = path.join(config.Config.FILE_DIRECTORY, file) # Analyze with olevba - vbaparser = oletools.olevba.VBA_Parser(filepath) - results = vbaparser.analyze_macros() + vbaparser = oletools.olevba.VBA_Parser(filename=filepath, relaxed=True) + stomping = vbaparser.detect_vba_stomping() + results = vbaparser.analyze_macros(show_decoded_strings=True, deobfuscate=True) + macros = vbaparser.extract_all_macros() + forms = vbaparser.find_vba_forms() + nb_macros = vbaparser.nb_macros + nb_autoexec = vbaparser.nb_autoexec + nb_iocs = vbaparser.nb_iocs + nb_suspicious = vbaparser.nb_suspicious - return jsonify({'filename': file, 'result': results}) + vbaparser.close() + return jsonify({ + "results": results, + "stomping": stomping, + "macros": macros, + "forms": forms, + "nb_macros": nb_macros, + "nb_autoexec": nb_autoexec, + "nb_iocs": nb_iocs, + "nb_suspicious": nb_suspicious + }) diff --git a/server/internal/database/msoffice.go b/server/internal/database/msoffice.go new file mode 100644 index 0000000..05bf1ed --- /dev/null +++ b/server/internal/database/msoffice.go @@ -0,0 +1,33 @@ +package database + +import ( + "context" + "log/slog" + + "git.jmbit.de/jmb/scanfile/server/internal/sqlc" + "github.com/jackc/pgx/v5/pgtype" +) + +func GetMSOfficeResults(fileID string) (sqlc.Msoffice, error) { + var pgUUID pgtype.UUID + err := pgUUID.Scan(fileID) + if err != nil { + slog.Error("Unable to convert string to UUID", "file-uuid", fileID, "error", err) + } + query := sqlc.New(pool) + data, err := query.GetMSOfficeResults(context.Background(), pgUUID) + if err != nil { + slog.Error("Error in GetMsofficeInfo", "file-uuid", fileID, "error", err) + } + return data, err +} + +func InsertMSOfficeResults(params sqlc.InsertMSOfficeResultsParams) error { + query := sqlc.New(pool) + slog.Debug("InsertMSOfficeResults", "params", params) + err := query.InsertMSOfficeResults(context.Background(), params) + if err != nil { + slog.Error("Error in InsertMsofficeInfo", "file-uuid", params.FileID.String(), "error", err) + } + return err +} diff --git a/server/internal/database/queries-file_properties.sql b/server/internal/database/queries-file_properties.sql index 0870854..faff843 100644 --- a/server/internal/database/queries-file_properties.sql +++ b/server/internal/database/queries-file_properties.sql @@ -12,18 +12,3 @@ INSERT INTO diec ( file_id, data ) VALUES ($1, $2); --- name: InsertFileMsofficeOleid :exec -INSERT INTO msoffice_oleid ( - file_id, data -) VALUES ($1, $2); - --- name: InsertFileMsofficeOlevba :exec -INSERT INTO msoffice_olevba ( - file_id, data -) VALUES ($1, $2); - --- name: InsertFileMsofficeMraptor :exec -INSERT INTO msoffice_mraptor ( - file_id, data -) VALUES ($1, $2); - diff --git a/server/internal/database/queries-msoffice.sql b/server/internal/database/queries-msoffice.sql new file mode 100644 index 0000000..6d3c1a8 --- /dev/null +++ b/server/internal/database/queries-msoffice.sql @@ -0,0 +1,34 @@ +-- name: InsertFileMsofficeOleid :exec +INSERT INTO msoffice_oleid ( + file_id, data +) VALUES ($1, $2); + +-- name: InsertFileMsofficeOlevba :exec +INSERT INTO msoffice_olevba ( + file_id, data +) VALUES ($1, $2); + +-- name: InsertFileMsofficeMraptor :exec +INSERT INTO msoffice_mraptor ( + file_id, data +) VALUES ($1, $2); + +-- name: GetMSOfficeData :one +SELECT t1.file_id, t1.data AS oleid, t2.data AS olevba, t3.data AS mraptor + FROM msoffice_oleid as t1 + LEFT join msoffice_olevba AS t2 ON t2.file_id = t1.file_id + LEFT JOIN msoffice_mraptor AS t3 ON t3.file_id = t1.file_id + WHERE t1.file_id = $1; + +-- name: InsertMSOfficeResults :exec +INSERT INTO msoffice ( + file_id, verdict, container_format, encrypted, file_format, vba_macros, xlm_macros, + vba_stomping, nb_autoexec, nb_iocs, nb_macros, nb_suspicious, olevba_results, macros +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14 +); + +-- name: GetMSOfficeResults :one +SELECT * FROM msoffice +WHERE file_id = $1 +LIMIT 1; diff --git a/server/internal/database/schema.sql b/server/internal/database/schema.sql index b2bdfb7..f202986 100644 --- a/server/internal/database/schema.sql +++ b/server/internal/database/schema.sql @@ -49,6 +49,25 @@ CREATE TABLE IF NOT EXISTS msoffice_mraptor ( data JSONB ); +CREATE TABLE IF NOT EXISTS msoffice ( + id BIGSERIAL PRIMARY KEY, + file_id UUID REFERENCES files (id) ON DELETE CASCADE, + verdict TEXT DEFAULT 'pending', + container_format TEXT, + encrypted BOOLEAN DEFAULT false, + file_format TEXT, + vba_macros TEXT, + xlm_macros TEXT, + vba_stomping BOOLEAN DEFAULT false, + nb_autoexec INTEGER, + nb_iocs INTEGER, + nb_macros INTEGER, + nb_suspicious INTEGER, + olevba_results JSONB, + macros JSONB +); + + CREATE TABLE IF NOT EXISTS file_properties ( id UUID PRIMARY KEY, sha256 BYTEA, @@ -65,5 +84,6 @@ CREATE INDEX idx_processing_jobs_file_id ON processing_jobs (file_id); CREATE INDEX idx_msoffice_oleid_file_id ON msoffice_oleid (file_id); CREATE INDEX idx_msoffice_olevba_file_id ON msoffice_olevba (file_id); CREATE INDEX idx_msoffice_mraptor_file_id ON msoffice_mraptor (file_id); +CREATE INDEX idx_msoffice_results_file_id ON msoffice_results (file_id); CREATE INDEX idx_file_properties_id ON file_properties (id); CREATE INDEX idx_file_id ON files (id); diff --git a/server/internal/processing/msoffice/mraptor.go b/server/internal/processing/msoffice/mraptor.go deleted file mode 100644 index 023676a..0000000 --- a/server/internal/processing/msoffice/mraptor.go +++ /dev/null @@ -1,41 +0,0 @@ -package msoffice - -import ( - "encoding/json" - "fmt" - "log/slog" - "net/http" - "net/url" - - "git.jmbit.de/jmb/scanfile/server/internal/database" - "github.com/jackc/pgx/v5/pgtype" - "github.com/spf13/viper" -) - -// MraptorScan() requests a scan of the file from the ole service -func MraptorScan(fileID pgtype.UUID) error { - slog.Debug("Starting MacroRaptor scan", "file-uuid", fileID.String()) - oleidUrl, err := url.Parse(viper.GetString("processing.oleurl")) - if err != nil { - slog.Error("Error parsing URL for ole service", "file-uuid", fileID.String(), "error", err) - } - oleidUrl.Path = "/olevba/analyze" - oleidUrl.Query().Add("file", fileID.String()) - oleidResp, err := http.Get(oleidUrl.String()) - slog.Debug("MraptorScan request", "file-uuid", fileID.String(), "url", oleidUrl.String(), "status-code", oleidResp.StatusCode) - if err != nil { - slog.Error("Error getting mraptor info from service", "file-uuid", fileID.String(), "error", err) - } - var body []byte - _, err = oleidResp.Body.Read(body) - if err != nil { - slog.Error("Error parsing mraptor body", "file-uuid", fileID.String(), "error", err) - } - - if json.Valid(body) == false { - return fmt.Errorf("JSON not valid") - } - slog.Debug("MraptorScan", "file-uuid", fileID.String(), "data", body) - database.InsertJsonResult(fileID, body, "msoffice_mraptor") - return nil -} diff --git a/server/internal/processing/msoffice/msoffice.go b/server/internal/processing/msoffice/msoffice.go index 58718ec..56c3aac 100644 --- a/server/internal/processing/msoffice/msoffice.go +++ b/server/internal/processing/msoffice/msoffice.go @@ -1,26 +1,58 @@ package msoffice import ( + "encoding/json" + "log/slog" + "git.jmbit.de/jmb/scanfile/server/internal/database" "git.jmbit.de/jmb/scanfile/server/internal/sqlc" ) func MSOfficeProcessing(job sqlc.ProcessingJob) error { database.StartProcessingJob(job.ID) - err := OleIDScan(job.FileID) + oleidResp, err := OleIDScan(job.FileID) if err != nil { database.FailProcessingJob(job.ID, err) return err } - err = OleVBAScan(job.FileID) + olevbaResp, err := OleVBAScan(job.FileID) if err != nil { database.FailProcessingJob(job.ID, err) return err } - err = MraptorScan(job.FileID) + params := sqlc.InsertMSOfficeResultsParams{ + FileID: job.FileID, + } + params.ContainerFormat.String = oleidResp.ContainerFormat + params.Encrypted.Bool = oleidResp.Encrypted + params.FileFormat.String = oleidResp.FileFormat + params.VbaMacros.String = oleidResp.VBAMacros + params.XlmMacros.String = oleidResp.XLMMacros + params.VbaStomping.Bool = olevbaResp.Stomping + params.NbAutoexec.Int32 = int32(olevbaResp.NbAutoexec) + params.NbIocs.Int32 = int32(olevbaResp.NbIocs) + params.NbMacros.Int32 = int32(olevbaResp.NbMacros) + params.NbSuspicious.Int32 = int32(olevbaResp.NbSuspicious) + + params.OlevbaResults, err = json.Marshal(olevbaResp.Results) + if err != nil { + slog.Error("Error in MSOfficeProcessing while marshaling olevba results to json", "file-uuid", job.FileID.String(), "error", err, "job-id", job.ID) + database.FailProcessingJob(job.ID, err) + return err + } + + params.Macros, err = json.Marshal(olevbaResp.Macros) + if err != nil { + slog.Error("Error in MSOfficeProcessing while marshaling macros to json", "file-uuid", job.FileID.String(), "error", err, "job-id", job.ID) + database.FailProcessingJob(job.ID, err) + return err + } + + err = database.InsertMSOfficeResults(params) if err != nil { database.FailProcessingJob(job.ID, err) return err } + return nil } diff --git a/server/internal/processing/msoffice/oleid.go b/server/internal/processing/msoffice/oleid.go index 98f6f32..33879d4 100644 --- a/server/internal/processing/msoffice/oleid.go +++ b/server/internal/processing/msoffice/oleid.go @@ -3,39 +3,53 @@ package msoffice import ( "encoding/json" "fmt" + "io" "log/slog" "net/http" "net/url" - "git.jmbit.de/jmb/scanfile/server/internal/database" "github.com/jackc/pgx/v5/pgtype" "github.com/spf13/viper" ) -func OleIDScan(fileID pgtype.UUID) error { +type oleidResponse struct { + ContainerFormat string `json:"Container format"` + Encrypted bool `json:"Encrypted"` + ExternalRelationships int `json:"External Relationships"` + FileFormat string `json:"File format"` + FlashObjects int `json:"Flash objects"` + ObjectPool bool `json:"ObjectPool"` + VBAMacros string `json:"VBA Macros"` + XLMMacros string `json:"XLM Macros"` +} + +func OleIDScan(fileID pgtype.UUID) (oleidResponse, error) { slog.Debug("Starting OleID scan", "file-uuid", fileID.String()) oleidUrl, err := url.Parse(viper.GetString("processing.oleurl")) if err != nil { - slog.Error("Error parsing URL for ole service", "file-uuid", fileID.String(), "error", err) + slog.Error("Error in OleIDScan parsing URL for ole service", "file-uuid", fileID.String(), "error", err) } oleidUrl.Path = "/oleid/analyze" - oleidUrl.Query().Add("file", fileID.String()) + oleidUrl.RawQuery = fmt.Sprintf("file=%s", fileID.String()) oleidResp, err := http.Get(oleidUrl.String()) slog.Debug("OleIDScan request", "file-uuid", fileID.String(), "url", oleidUrl.String(), "status-code", oleidResp.StatusCode) if err != nil { - slog.Error("Error getting oleid info from service", "file-uuid", fileID.String(), "error", err) + slog.Error("Error in OleIDScan getting oleid info from service", "file-uuid", fileID.String(), "error", err) } - var body []byte - _, err = oleidResp.Body.Read(body) + defer oleidResp.Body.Close() + body, err := io.ReadAll(oleidResp.Body) if err != nil { - slog.Error("Error parsing oleid body", "file-uuid", fileID.String(), "error", err) + slog.Error("Error in OleIDScan parsing oleid body", "file-uuid", fileID.String(), "error", err) } + var jsonResponse oleidResponse + + err = json.Unmarshal(body, &jsonResponse) + if err != nil { + slog.Error("Error in OleIDScan when trying to unmarshal response", "file-uuid", fileID.String(), "error", err) + return jsonResponse, err + } - if json.Valid(body) == false { - return fmt.Errorf("JSON not valid") - } slog.Debug("OleIDScan", "file-uuid", fileID.String(), "data", body) - database.InsertJsonResult(fileID, body, "msoffice_oleid") - return nil + return jsonResponse, nil } diff --git a/server/internal/processing/msoffice/olevba.go b/server/internal/processing/msoffice/olevba.go index 052bed8..59fb49e 100644 --- a/server/internal/processing/msoffice/olevba.go +++ b/server/internal/processing/msoffice/olevba.go @@ -3,38 +3,70 @@ package msoffice import ( "encoding/json" "fmt" + "io" "log/slog" "net/http" "net/url" + "strings" - "git.jmbit.de/jmb/scanfile/server/internal/database" "github.com/jackc/pgx/v5/pgtype" "github.com/spf13/viper" ) -func OleVBAScan(fileID pgtype.UUID) error { +type olevbaResponse struct { + Forms any `json:"forms"` + Macros [][]string `json:"macros"` + NbAutoexec int `json:"nb_autoexec"` + NbIocs int `json:"nb_iocs"` + NbMacros int `json:"nb_macros"` + NbSuspicious int `json:"nb_suspicious"` + Results [][]string `json:"results"` + Stomping bool `json:"stomping"` +} + + +func OleVBAScan(fileID pgtype.UUID) (olevbaResponse, error) { slog.Debug("Starting OLEvba scan", "file-uuid", fileID.String()) oleidUrl, err := url.Parse(viper.GetString("processing.oleurl")) if err != nil { - slog.Error("Error parsing URL for ole service", "file-uuid", fileID.String(), "error", err) + slog.Error("Error in OleVBAScan parsing URL for ole service", "file-uuid", fileID.String(), "error", err) } oleidUrl.Path = "/olevba/analyze" - oleidUrl.Query().Add("file", fileID.String()) + oleidUrl.RawQuery = fmt.Sprintf("file=%s", fileID.String()) oleidResp, err := http.Get(oleidUrl.String()) slog.Debug("OleVBAScan request", "file-uuid", fileID.String(), "url", oleidUrl.String(), "status-code", oleidResp.StatusCode) if err != nil { - slog.Error("Error getting olevba info from service", "file-uuid", fileID.String(), "error", err) + slog.Error("Error in OleVBAScan getting olevba info from service", "file-uuid", fileID.String(), "error", err) } - var body []byte - _, err = oleidResp.Body.Read(body) + defer oleidResp.Body.Close() + body, err := io.ReadAll(oleidResp.Body) if err != nil { - slog.Error("Error parsing olevba body", "file-uuid", fileID.String(), "error", err) + slog.Error("Error in OleVBAScan parsing olevba body", "file-uuid", fileID.String(), "error", err) } - if json.Valid(body) == false { - return fmt.Errorf("JSON not valid") - } - slog.Debug("OleVBAScan", "file-uuid", fileID.String(), "data", body) - database.InsertJsonResult(fileID, body, "msoffice_olevba") - return nil + var jsonResp olevbaResponse + + err = json.Unmarshal(body, &jsonResp) + if err != nil { + slog.Error("Error in OleVBAScan when trying to unmarshal response", "file-uuid", fileID.String(), "error", err) + return jsonResp, err + } + + for i, result := range jsonResp.Results { + if result[0] == "Hex String" { + var hexParts []string + for _, b := range []byte(result[1]) { + hexParts = append(hexParts, fmt.Sprintf("0x%X", b)) + } + result[1] = strings.Join(hexParts, " ") + } + slog.Debug("OleVBAScan Result", "0", result[0], "1", result[1], "2", result[2], "i", i) + } + + for i, macro := range jsonResp.Macros { + slog.Debug("OleVBAScan Macro", "0", macro[0], "1", macro[1], "2", macro[2], "3", macro[3], "i", i) + } + + slog.Debug("OleVBAScan", "file-uuid", fileID.String(), "data", jsonResp) + return jsonResp, nil } diff --git a/server/internal/processing/msoffice/olevba_test.go b/server/internal/processing/msoffice/olevba_test.go new file mode 100644 index 0000000..1783566 --- /dev/null +++ b/server/internal/processing/msoffice/olevba_test.go @@ -0,0 +1,26 @@ +package msoffice_test + +import ( + "log/slog" + "os" + "testing" + + "git.jmbit.de/jmb/scanfile/server/internal/processing/msoffice" + "github.com/jackc/pgx/v5/pgtype" + "github.com/spf13/viper" +) + + +func TestOleVba(t *testing.T) { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))) + viper.Set("processing.oleurl", "http://localhost:5000") + var fileid pgtype.UUID + fileid.Scan("cf645d68-fc5b-4cba-8940-4ccce437e354") + t.Log(fileid) + resp, err := msoffice.OleVBAScan(fileid) + if err != nil { + t.FailNow() + } + t.Log(resp) + +}