From 8de46a3682f2391b48a0138c5f98d583cb2297df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20B=C3=BClow?= Date: Wed, 23 Jul 2025 13:19:28 +0200 Subject: [PATCH] yara stuff --- server/internal/config/config.go | 5 +- server/internal/database/queries-yara.sql | 14 ++++++ server/internal/database/schema.sql | 8 +++- server/internal/database/yara.go | 33 +++++++++++++ server/internal/processing/processing.go | 2 + server/internal/processing/yara/wrap.go | 56 +++++++++++++++++++++++ server/internal/processing/yara/yara.go | 32 +++++++++++++ server/internal/sqlc/models.go | 8 +++- server/internal/sqlc/queries-yara.sql.go | 54 ++++++++++++++++++++++ sqlc.yaml | 1 + 10 files changed, 209 insertions(+), 4 deletions(-) create mode 100644 server/internal/database/queries-yara.sql create mode 100644 server/internal/database/yara.go create mode 100644 server/internal/processing/yara/wrap.go create mode 100644 server/internal/processing/yara/yara.go create mode 100644 server/internal/sqlc/queries-yara.sql.go diff --git a/server/internal/config/config.go b/server/internal/config/config.go index c225194..118a523 100644 --- a/server/internal/config/config.go +++ b/server/internal/config/config.go @@ -3,7 +3,6 @@ package config import ( "log/slog" - _ "github.com/joho/godotenv/autoload" "github.com/spf13/viper" ) @@ -47,7 +46,9 @@ func setDefaults() { // Others viper.SetDefault("processing.oleurl", "http://localhost:5000") viper.SetDefault("processing.maxmimesize", "100MB") - viper.SetDefault("store.path", "./storage/files/") + viper.SetDefault("processing.yararules", "./storage/rules") + viper.SetDefault("processing.yaracompiled", "./storage/output.yarc") + viper.SetDefault("store.path", "./storage/files") viper.SetDefault("debug", false) // UI Interface info viper.SetDefault("ui.name", "Scanfile") diff --git a/server/internal/database/queries-yara.sql b/server/internal/database/queries-yara.sql new file mode 100644 index 0000000..59721f6 --- /dev/null +++ b/server/internal/database/queries-yara.sql @@ -0,0 +1,14 @@ +-- name: InsertYaraResults :one +INSERT INTO yara_results ( + file_id, matched +) VALUES ($1, $2) +RETURNING *; + +-- name: GetYaraResults :one +SELECT * FROM yara_results +WHERE file_id = $1 +LIMIT 1; + +-- name: DeleteYaraResults :exec +DELETE FROM yara_results +WHERE id = $1; diff --git a/server/internal/database/schema.sql b/server/internal/database/schema.sql index 7c0891c..9661278 100644 --- a/server/internal/database/schema.sql +++ b/server/internal/database/schema.sql @@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS processing_jobs ( status TEXT, job_type TEXT, error TEXT, - messages JSONB DEFAULT '[]'::JSONB + messages TEXT[] ); CREATE TABLE IF NOT EXISTS diec ( @@ -77,6 +77,12 @@ CREATE TABLE IF NOT EXISTS file_properties ( libmagic_apple TEXT ); +CREATE TABLE IF NOT EXISTS yara_results ( + id BIGSERIAL PRIMARY KEY, + file_id UUID REFERENCES files (id) ON DELETE CASCADE, + matched TEXT[] +); + -- Indices -- Since tables will be heavily accessed by file_id, there should be indices for them CREATE INDEX idx_diec_file_id ON diec (file_id); diff --git a/server/internal/database/yara.go b/server/internal/database/yara.go new file mode 100644 index 0000000..10184b9 --- /dev/null +++ b/server/internal/database/yara.go @@ -0,0 +1,33 @@ +package database + +import ( + "context" + "log/slog" + + "git.jmbit.de/jmb/scanfile/server/internal/sqlc" + "github.com/jackc/pgx/v5/pgtype" +) + +func GetYaraResults(fileID string) (sqlc.YaraResult, error) { + var pgUUID pgtype.UUID + err := pgUUID.Scan(fileID) + if err != nil { + slog.Error("Unable to convert string to UUID", "file-uuid", fileID, "error", err) + } + query := sqlc.New(pool) + data, err := query.GetYaraResults(context.Background(), pgUUID) + if err != nil { + slog.Error("Error in GetMsofficeInfo", "file-uuid", fileID, "error", err) + } + return data, err +} + +func InsertYaraResults(params sqlc.InsertYaraResultsParams) error { + query := sqlc.New(pool) + slog.Debug("InsertYaraResults", "params", params) + _, err := query.InsertYaraResults(context.Background(), params) + if err != nil { + slog.Error("Error in InsertYaraResults", "file-uuid", params.FileID.String(), "error", err) + } + return err +} diff --git a/server/internal/processing/processing.go b/server/internal/processing/processing.go index 2e7b53f..55f06d5 100644 --- a/server/internal/processing/processing.go +++ b/server/internal/processing/processing.go @@ -9,6 +9,7 @@ import ( "git.jmbit.de/jmb/scanfile/server/internal/database" "git.jmbit.de/jmb/scanfile/server/internal/processing/basic" "git.jmbit.de/jmb/scanfile/server/internal/processing/msoffice" + "git.jmbit.de/jmb/scanfile/server/internal/processing/yara" "github.com/jackc/pgx/v5/pgtype" ) @@ -17,6 +18,7 @@ var startup time.Time func Setup(wg *sync.WaitGroup) { startup = time.Now() + yara.InitYara() } // Submit() starts the analysis process for a file. diff --git a/server/internal/processing/yara/wrap.go b/server/internal/processing/yara/wrap.go new file mode 100644 index 0000000..2fcde22 --- /dev/null +++ b/server/internal/processing/yara/wrap.go @@ -0,0 +1,56 @@ +package yara + +import ( + "log/slog" + "os/exec" + "path/filepath" + + "git.jmbit.de/jmb/scanfile/server/internal/store" + "github.com/spf13/viper" +) + +func compileSourcesFromFiles() error { + root, err := filepath.Abs(viper.GetString("processing.yararules")) + if err != nil { + slog.Error("Error getting absolute path for processing.yararules", "error", err) + return err + } + outputPath, err := filepath.Abs(viper.GetString("processing.yaracompiled")) + if err != nil { + slog.Error("Error getting absolute path for processing.yaracompiled", "error", err) + return err + } + + cmd := exec.Command("/usr/local/bin/yr", "compile","-path-as-namespace", "--relaxed-re-syntax", "--output", outputPath, root) + result, err := cmd.Output() + if err != nil { + slog.Error("Error compiling yara rules", "error", err, "result", string(result)) + return err + } else { + slog.Info("Compiled yara rules", "result", string(result)) + } + + return nil +} + +func scanFile(fileName string) ([]string, error) { + var matched []string + outputPath, err := filepath.Abs(viper.GetString("processing.yaracompiled")) + if err != nil { + slog.Error("Error getting absolute path for processing.yaracompiled", "error", err) + return matched, err + } + fullPath, err := store.AbsPath(fileName) + if err != nil { + slog.Error("Error in DiecScan", "file-uuid", fileName, "error", err) + return matched, err + } + cmd := exec.Command("/usr/local/bin/yr", "scan", "--output-format ndjson", "--print-namespace","--compiled-rules", outputPath, fullPath) + result, err := cmd.Output() + if err != nil { + slog.Error("Error scanning file with yara", "error", err, "file-uuid", fileName,"result", string(result)) + return matched, err + } + + return matched, nil +} diff --git a/server/internal/processing/yara/yara.go b/server/internal/processing/yara/yara.go new file mode 100644 index 0000000..8b97c5d --- /dev/null +++ b/server/internal/processing/yara/yara.go @@ -0,0 +1,32 @@ +package yara + +import ( + "git.jmbit.de/jmb/scanfile/server/internal/database" + "git.jmbit.de/jmb/scanfile/server/internal/sqlc" +) + +func InitYara() error{ + return compileSourcesFromFiles() +} + +func YaraProcessing(job sqlc.ProcessingJob) error { + database.StartProcessingJob(job.ID) + results, err := scanFile(job.FileID.String()) + if err != nil { + database.FailProcessingJob(job.ID, err) + return err + } + + params := sqlc.InsertYaraResultsParams{ + FileID: job.FileID, + Matched: results, + } + err = database.InsertYaraResults(params) + if err != nil { + database.FailProcessingJob(job.ID, err) + return err + } + + + return nil +} diff --git a/server/internal/sqlc/models.go b/server/internal/sqlc/models.go index f4dd3c7..ed23421 100644 --- a/server/internal/sqlc/models.go +++ b/server/internal/sqlc/models.go @@ -79,5 +79,11 @@ type ProcessingJob struct { Status pgtype.Text JobType pgtype.Text Error pgtype.Text - Messages []byte + Messages []string +} + +type YaraResult struct { + ID int64 + FileID pgtype.UUID + Matched []string } diff --git a/server/internal/sqlc/queries-yara.sql.go b/server/internal/sqlc/queries-yara.sql.go new file mode 100644 index 0000000..471df43 --- /dev/null +++ b/server/internal/sqlc/queries-yara.sql.go @@ -0,0 +1,54 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: queries-yara.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const deleteYaraResults = `-- name: DeleteYaraResults :exec +DELETE FROM yara_results +WHERE id = $1 +` + +func (q *Queries) DeleteYaraResults(ctx context.Context, id int64) error { + _, err := q.db.Exec(ctx, deleteYaraResults, id) + return err +} + +const getYaraResults = `-- name: GetYaraResults :one +SELECT id, file_id, matched FROM yara_results +WHERE file_id = $1 +LIMIT 1 +` + +func (q *Queries) GetYaraResults(ctx context.Context, fileID pgtype.UUID) (YaraResult, error) { + row := q.db.QueryRow(ctx, getYaraResults, fileID) + var i YaraResult + err := row.Scan(&i.ID, &i.FileID, &i.Matched) + return i, err +} + +const insertYaraResults = `-- name: InsertYaraResults :one +INSERT INTO yara_results ( + file_id, matched +) VALUES ($1, $2) +RETURNING id, file_id, matched +` + +type InsertYaraResultsParams struct { + FileID pgtype.UUID + Matched []string +} + +func (q *Queries) InsertYaraResults(ctx context.Context, arg InsertYaraResultsParams) (YaraResult, error) { + row := q.db.QueryRow(ctx, insertYaraResults, arg.FileID, arg.Matched) + var i YaraResult + err := row.Scan(&i.ID, &i.FileID, &i.Matched) + return i, err +} diff --git a/sqlc.yaml b/sqlc.yaml index 88e73db..d59880a 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -8,6 +8,7 @@ sql: - "server/internal/database/queries-file_properties.sql" - "server/internal/database/queries-processing_jobs.sql" - "server/internal/database/queries-msoffice.sql" + - "server/internal/database/queries-yara.sql" database: managed: false uri: "postgresql://scanfile:${PG_PASSWORD}@localhost:5432/scanfile"