yara stuff

server/internal/config/config.go

@@ -3,7 +3,6 @@ package config
 import (
 	"log/slog"
 
-	_ "github.com/joho/godotenv/autoload"
 	"github.com/spf13/viper"
 )
 
@@ -47,7 +46,9 @@ func setDefaults() {
 	// Others
 	viper.SetDefault("processing.oleurl", "http://localhost:5000")
   viper.SetDefault("processing.maxmimesize", "100MB")
-	viper.SetDefault("store.path", "./storage/files/")
+  viper.SetDefault("processing.yararules", "./storage/rules")
+	viper.SetDefault("processing.yaracompiled", "./storage/output.yarc")
+	viper.SetDefault("store.path", "./storage/files")
 	viper.SetDefault("debug", false)
   // UI Interface info
   viper.SetDefault("ui.name", "Scanfile")

server/internal/database/queries-yara.sql

@@ -0,0 +1,14 @@
+-- name: InsertYaraResults :one
+INSERT INTO yara_results (
+ file_id, matched
+) VALUES ($1, $2)
+RETURNING *;
+
+-- name: GetYaraResults :one
+SELECT * FROM yara_results
+WHERE file_id = $1
+LIMIT 1;
+
+-- name: DeleteYaraResults :exec
+DELETE FROM yara_results
+WHERE id = $1;

server/internal/database/schema.sql

@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS processing_jobs (
   status TEXT,
   job_type TEXT,
   error TEXT,
-  messages JSONB DEFAULT '[]'::JSONB
+  messages TEXT[]
 );
 
 CREATE TABLE IF NOT EXISTS diec (
@@ -77,6 +77,12 @@ CREATE TABLE IF NOT EXISTS file_properties (
   libmagic_apple TEXT
 );
 
+CREATE TABLE IF NOT EXISTS yara_results (
+  id BIGSERIAL PRIMARY KEY,
+  file_id UUID REFERENCES files (id) ON DELETE CASCADE,
+  matched TEXT[]
+);
+
 -- Indices
 -- Since tables will be heavily accessed by file_id, there should be indices for them
 CREATE INDEX idx_diec_file_id ON diec (file_id);

server/internal/database/yara.go

@@ -0,0 +1,33 @@
+package database
+
+import (
+	"context"
+	"log/slog"
+
+	"git.jmbit.de/jmb/scanfile/server/internal/sqlc"
+	"github.com/jackc/pgx/v5/pgtype"
+)
+
+func GetYaraResults(fileID string) (sqlc.YaraResult, error) {
+	var pgUUID pgtype.UUID
+	err := pgUUID.Scan(fileID)
+	if err != nil {
+		slog.Error("Unable to convert string to UUID", "file-uuid", fileID, "error", err)
+	}
+	query := sqlc.New(pool)
+  data, err := query.GetYaraResults(context.Background(), pgUUID)
+  if err != nil {
+    slog.Error("Error in GetMsofficeInfo", "file-uuid", fileID, "error", err)
+  }
+  return data, err
+}
+
+func InsertYaraResults(params sqlc.InsertYaraResultsParams) error {
+  query := sqlc.New(pool)
+  slog.Debug("InsertYaraResults", "params", params)
+  _, err := query.InsertYaraResults(context.Background(), params)
+  if err != nil {
+    slog.Error("Error in InsertYaraResults", "file-uuid", params.FileID.String(), "error", err)
+  }
+  return err
+}

server/internal/processing/processing.go

@@ -9,6 +9,7 @@ import (
 	"git.jmbit.de/jmb/scanfile/server/internal/database"
 	"git.jmbit.de/jmb/scanfile/server/internal/processing/basic"
 	"git.jmbit.de/jmb/scanfile/server/internal/processing/msoffice"
+	"git.jmbit.de/jmb/scanfile/server/internal/processing/yara"
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
@@ -17,6 +18,7 @@ var startup time.Time
 
 func Setup(wg *sync.WaitGroup) {
 	startup = time.Now()
+  yara.InitYara()
 }
 
 // Submit() starts the analysis process for a file.

server/internal/processing/yara/wrap.go

@@ -0,0 +1,56 @@
+package yara
+
+import (
+	"log/slog"
+	"os/exec"
+	"path/filepath"
+
+	"git.jmbit.de/jmb/scanfile/server/internal/store"
+	"github.com/spf13/viper"
+)
+
+func compileSourcesFromFiles() error {
+  root, err := filepath.Abs(viper.GetString("processing.yararules"))
+  if err != nil {
+    slog.Error("Error getting absolute path for processing.yararules", "error", err)
+    return err
+  }
+	outputPath, err := filepath.Abs(viper.GetString("processing.yaracompiled"))
+	if err != nil {
+    slog.Error("Error getting absolute path for processing.yaracompiled", "error", err)
+    return err
+	}
+
+	cmd := exec.Command("/usr/local/bin/yr", "compile","-path-as-namespace", "--relaxed-re-syntax", "--output", outputPath, root)
+	result, err := cmd.Output()
+	if err != nil {
+		slog.Error("Error compiling yara rules", "error", err, "result", string(result))
+		return err
+	} else {
+		slog.Info("Compiled yara rules", "result", string(result))
+	}
+	
+	return nil
+}
+
+func scanFile(fileName string) ([]string, error) {
+	var matched []string
+	outputPath, err := filepath.Abs(viper.GetString("processing.yaracompiled"))
+	if err != nil {
+    slog.Error("Error getting absolute path for processing.yaracompiled", "error", err)
+    return matched, err
+	}
+	fullPath, err := store.AbsPath(fileName)
+	if err != nil {
+		slog.Error("Error in DiecScan", "file-uuid", fileName, "error", err)
+		return matched, err
+	}
+	cmd := exec.Command("/usr/local/bin/yr", "scan", "--output-format ndjson", "--print-namespace","--compiled-rules", outputPath, fullPath)
+	result, err := cmd.Output()
+	if err != nil {
+		slog.Error("Error scanning file with yara", "error", err, "file-uuid", fileName,"result", string(result))
+		return matched, err
+	}
+
+	return matched, nil
+}

server/internal/processing/yara/yara.go

@@ -0,0 +1,32 @@
+package yara
+
+import (
+	"git.jmbit.de/jmb/scanfile/server/internal/database"
+	"git.jmbit.de/jmb/scanfile/server/internal/sqlc"
+)
+
+func InitYara() error{
+	return compileSourcesFromFiles()
+}
+
+func YaraProcessing(job sqlc.ProcessingJob) error {
+	database.StartProcessingJob(job.ID)
+  results, err := scanFile(job.FileID.String())
+  if err != nil {
+    database.FailProcessingJob(job.ID, err)
+		return err
+  }
+
+	params := sqlc.InsertYaraResultsParams{
+		FileID: job.FileID,
+		Matched: results,
+	}
+	err = database.InsertYaraResults(params)
+  if err != nil {
+    database.FailProcessingJob(job.ID, err)
+		return err
+  }
+
+
+  return nil
+}

server/internal/sqlc/models.go

@@ -79,5 +79,11 @@ type ProcessingJob struct {
 	Status    pgtype.Text
 	JobType   pgtype.Text
 	Error     pgtype.Text
-	Messages  []byte
+	Messages  []string
+}
+
+type YaraResult struct {
+	ID      int64
+	FileID  pgtype.UUID
+	Matched []string
 }

server/internal/sqlc/queries-yara.sql.go

@@ -0,0 +1,54 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.29.0
+// source: queries-yara.sql
+
+package sqlc
+
+import (
+	"context"
+
+	"github.com/jackc/pgx/v5/pgtype"
+)
+
+const deleteYaraResults = `-- name: DeleteYaraResults :exec
+DELETE FROM yara_results
+WHERE id = $1
+`
+
+func (q *Queries) DeleteYaraResults(ctx context.Context, id int64) error {
+	_, err := q.db.Exec(ctx, deleteYaraResults, id)
+	return err
+}
+
+const getYaraResults = `-- name: GetYaraResults :one
+SELECT id, file_id, matched FROM yara_results
+WHERE file_id = $1
+LIMIT 1
+`
+
+func (q *Queries) GetYaraResults(ctx context.Context, fileID pgtype.UUID) (YaraResult, error) {
+	row := q.db.QueryRow(ctx, getYaraResults, fileID)
+	var i YaraResult
+	err := row.Scan(&i.ID, &i.FileID, &i.Matched)
+	return i, err
+}
+
+const insertYaraResults = `-- name: InsertYaraResults :one
+INSERT INTO yara_results (
+ file_id, matched
+) VALUES ($1, $2)
+RETURNING id, file_id, matched
+`
+
+type InsertYaraResultsParams struct {
+	FileID  pgtype.UUID
+	Matched []string
+}
+
+func (q *Queries) InsertYaraResults(ctx context.Context, arg InsertYaraResultsParams) (YaraResult, error) {
+	row := q.db.QueryRow(ctx, insertYaraResults, arg.FileID, arg.Matched)
+	var i YaraResult
+	err := row.Scan(&i.ID, &i.FileID, &i.Matched)
+	return i, err
+}

sqlc.yaml

@@ -8,6 +8,7 @@ sql:
       - "server/internal/database/queries-file_properties.sql"
       - "server/internal/database/queries-processing_jobs.sql"
       - "server/internal/database/queries-msoffice.sql"
+      - "server/internal/database/queries-yara.sql"
     database:
       managed: false
       uri: "postgresql://scanfile:${PG_PASSWORD}@localhost:5432/scanfile"