sql stuff

server/internal/database/schema.sql

@@ -2,6 +2,17 @@
 CREATE EXTENSION IF NOT EXISTS "pgcrypto";
 
 -- Tables
+
+CREATE TABLE IF NOT EXISTS users (
+  id BIGSERIAL PRIMARY KEY,
+  user_name TEXT NOT NULL UNIQUE,
+  display_name TEXT NOT NULL,
+  pw_hash BYTEA NOT NULL,
+  email_address TEXT NOT NULL,
+  enabled BOOLEAN NOT NULL DEFAULT TRUE,
+  created TIMESTAMP DEFAULT NOW() NOT NULL
+);
+
 CREATE TABLE IF NOT EXISTS files (
   id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
   name TEXT NOT NULL,
@@ -9,6 +20,8 @@ CREATE TABLE IF NOT EXISTS files (
   mimetype TEXT NOT NULL,
   size BIGINT NOT NULL,
   blake2 BYTEA NOT NULL UNIQUE,
+  score DECIMAL,
+  creator BIGINT REFERENCES users (id),
   created TIMESTAMP DEFAULT NOW() NOT NULL,
   updated TIMESTAMP DEFAULT NOW() NOT NULL
 );
@@ -16,7 +29,7 @@ CREATE TABLE IF NOT EXISTS files (
 CREATE TABLE IF NOT EXISTS processing_jobs (
   id BIGSERIAL PRIMARY KEY,
   file_id UUID REFERENCES files (id) ON DELETE CASCADE,
-  created TIMESTAMP DEFAULT NOW(),
+  created TIMESTAMP DEFAULT NOW() NOT NULL,
   started TIMESTAMP,
   completed TIMESTAMP,
   status TEXT,
@@ -28,7 +41,8 @@ CREATE TABLE IF NOT EXISTS processing_jobs (
 CREATE TABLE IF NOT EXISTS diec (
   id BIGSERIAL PRIMARY KEY,
   file_id UUID REFERENCES files (id) ON DELETE CASCADE,
-  data JSONB
+  data JSONB,
+  created TIMESTAMP DEFAULT NOW() NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS msoffice (
@@ -46,7 +60,8 @@ CREATE TABLE IF NOT EXISTS msoffice (
   nb_macros INTEGER,
   nb_suspicious INTEGER,
   olevba_results TEXT[][],
-  macros TEXT[][]
+  macros TEXT[][],
+  created TIMESTAMP DEFAULT NOW() NOT NULL
 );
 
 
@@ -57,23 +72,32 @@ CREATE TABLE IF NOT EXISTS file_properties (
   md5 BYTEA,
   libmagic_mime TEXT,
   libmagic_extension TEXT,
-  libmagic_apple TEXT
+  libmagic_apple TEXT,
+  created TIMESTAMP DEFAULT NOW() NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS yara_results (
   id BIGSERIAL PRIMARY KEY,
   file_id UUID REFERENCES files (id) ON DELETE CASCADE,
-  matched TEXT[]
+  matched TEXT[],
+  created TIMESTAMP DEFAULT NOW() NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS capa_results (
+  id BIGSERIAL PRIMARY KEY,
+  file_id UUID REFERENCES files (id) ON DELETE CASCADE,
+  data JSONB,
+  type TEXT DEFAULT 'default' NOT NULL,
+  created TIMESTAMP DEFAULT NOW() NOT NULL
 );
 
 -- Indices
 -- Since tables will be heavily accessed by file_id, there should be indices for them
 CREATE INDEX idx_diec_file_id ON diec (file_id);
 CREATE INDEX idx_processing_jobs_file_id ON processing_jobs (file_id);
-CREATE INDEX idx_msoffice_oleid_file_id ON msoffice_oleid (file_id);
-CREATE INDEX idx_msoffice_olevba_file_id ON msoffice_olevba (file_id);
-CREATE INDEX idx_msoffice_mraptor_file_id ON msoffice_mraptor (file_id);
 CREATE INDEX idx_msoffice_results_file_id ON msoffice (file_id);
 CREATE INDEX idx_file_properties_file_id ON file_properties (file_id);
 CREATE INDEX idx_file_id ON files (id);
 CREATE INDEX idx_yara_results_file_id ON yara_results (file_id);
+CREATE INDEX idx_user_name ON users (user_name);
+CREATE INDEX idx_capa_results_file_id ON capa_results (file_id);

server/internal/sqlc/models.go

@@ -8,10 +8,19 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
+type CapaResult struct {
+	ID      int64
+	FileID  pgtype.UUID
+	Data    []byte
+	Type    string
+	Created pgtype.Timestamp
+}
+
 type Diec struct {
-	ID     int64
+	ID      int64
-	FileID pgtype.UUID
+	FileID  pgtype.UUID
-	Data   []byte
+	Data    []byte
+	Created pgtype.Timestamp
 }
 
 type File struct {
@@ -21,6 +30,8 @@ type File struct {
 	Mimetype    string
 	Size        int64
 	Blake2      []byte
+	Score       pgtype.Numeric
+	Creator     pgtype.Int8
 	Created     pgtype.Timestamp
 	Updated     pgtype.Timestamp
 }
@@ -33,6 +44,7 @@ type FileProperty struct {
 	LibmagicMime      pgtype.Text
 	LibmagicExtension pgtype.Text
 	LibmagicApple     pgtype.Text
+	Created           pgtype.Timestamp
 }
 
 type Msoffice struct {
@@ -51,6 +63,7 @@ type Msoffice struct {
 	NbSuspicious    pgtype.Int4
 	OlevbaResults   [][]string
 	Macros          [][]string
+	Created         pgtype.Timestamp
 }
 
 type ProcessingJob struct {
@@ -65,8 +78,19 @@ type ProcessingJob struct {
 	Messages  []string
 }
 
+type User struct {
+	ID           int64
+	UserName     string
+	DisplayName  string
+	PwHash       []byte
+	EmailAddress string
+	Enabled      bool
+	Created      pgtype.Timestamp
+}
+
 type YaraResult struct {
 	ID      int64
 	FileID  pgtype.UUID
 	Matched []string
+	Created pgtype.Timestamp
 }

server/internal/sqlc/queries-file_properties.sql.go

@@ -11,8 +11,43 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
+const getFileCapa = `-- name: GetFileCapa :one
+SELECT id, file_id, data, type, created FROM capa_results
+WHERE file_id = $1
+`
+
+func (q *Queries) GetFileCapa(ctx context.Context, fileID pgtype.UUID) (CapaResult, error) {
+	row := q.db.QueryRow(ctx, getFileCapa, fileID)
+	var i CapaResult
+	err := row.Scan(
+		&i.ID,
+		&i.FileID,
+		&i.Data,
+		&i.Type,
+		&i.Created,
+	)
+	return i, err
+}
+
+const getFileDIEC = `-- name: GetFileDIEC :one
+SELECT id, file_id, data, created FROM diec
+WHERE file_id = $1
+`
+
+func (q *Queries) GetFileDIEC(ctx context.Context, fileID pgtype.UUID) (Diec, error) {
+	row := q.db.QueryRow(ctx, getFileDIEC, fileID)
+	var i Diec
+	err := row.Scan(
+		&i.ID,
+		&i.FileID,
+		&i.Data,
+		&i.Created,
+	)
+	return i, err
+}
+
 const getFileProperties = `-- name: GetFileProperties :one
-SELECT id, file_id, sha256, md5, libmagic_mime, libmagic_extension, libmagic_apple FROM file_properties
+SELECT id, file_id, sha256, md5, libmagic_mime, libmagic_extension, libmagic_apple, created FROM file_properties
 WHERE file_id = $1
 `
 
@@ -27,10 +62,27 @@ func (q *Queries) GetFileProperties(ctx context.Context, fileID pgtype.UUID) (Fi
 		&i.LibmagicMime,
 		&i.LibmagicExtension,
 		&i.LibmagicApple,
+		&i.Created,
 	)
 	return i, err
 }
 
+const insertFileCapa = `-- name: InsertFileCapa :exec
+INSERT INTO capa_results (
+ file_id, data
+) VALUES ($1, $2)
+`
+
+type InsertFileCapaParams struct {
+	FileID pgtype.UUID
+	Data   []byte
+}
+
+func (q *Queries) InsertFileCapa(ctx context.Context, arg InsertFileCapaParams) error {
+	_, err := q.db.Exec(ctx, insertFileCapa, arg.FileID, arg.Data)
+	return err
+}
+
 const insertFileDIEC = `-- name: InsertFileDIEC :exec
 INSERT INTO diec (
   file_id, data

server/internal/sqlc/queries-files.sql.go

@@ -18,7 +18,7 @@ INSERT INTO files (
   $1,$2,$3,$4,$5
 )
 ON CONFLICT DO NOTHING -- Handle this in application code
-RETURNING id, name, description, mimetype, size, blake2, created, updated
+RETURNING id, name, description, mimetype, size, blake2, score, creator, created, updated
 `
 
 type CreateFileParams struct {
@@ -45,6 +45,8 @@ func (q *Queries) CreateFile(ctx context.Context, arg CreateFileParams) (File, e
 		&i.Mimetype,
 		&i.Size,
 		&i.Blake2,
+		&i.Score,
+		&i.Creator,
 		&i.Created,
 		&i.Updated,
 	)
@@ -63,7 +65,7 @@ func (q *Queries) DeleteFile(ctx context.Context, id pgtype.UUID) error {
 }
 
 const getAllFiles = `-- name: GetAllFiles :many
-SELECT id, name, description, mimetype, size, blake2, created, updated FROM files
+SELECT id, name, description, mimetype, size, blake2, score, creator, created, updated FROM files
 ORDER BY created DESC
 `
 
@@ -83,6 +85,8 @@ func (q *Queries) GetAllFiles(ctx context.Context) ([]File, error) {
 			&i.Mimetype,
 			&i.Size,
 			&i.Blake2,
+			&i.Score,
+			&i.Creator,
 			&i.Created,
 			&i.Updated,
 		); err != nil {
@@ -97,7 +101,7 @@ func (q *Queries) GetAllFiles(ctx context.Context) ([]File, error) {
 }
 
 const getFileByBlake2 = `-- name: GetFileByBlake2 :one
-SELECT id, name, description, mimetype, size, blake2, created, updated
+SELECT id, name, description, mimetype, size, blake2, score, creator, created, updated
 FROM files
 WHERE blake2 = $1
 `
@@ -112,6 +116,8 @@ func (q *Queries) GetFileByBlake2(ctx context.Context, blake2 []byte) (File, err
 		&i.Mimetype,
 		&i.Size,
 		&i.Blake2,
+		&i.Score,
+		&i.Creator,
 		&i.Created,
 		&i.Updated,
 	)
@@ -119,7 +125,7 @@ func (q *Queries) GetFileByBlake2(ctx context.Context, blake2 []byte) (File, err
 }
 
 const getFileByUUID = `-- name: GetFileByUUID :one
-SELECT id, name, description, mimetype, size, blake2, created, updated 
+SELECT id, name, description, mimetype, size, blake2, score, creator, created, updated 
 FROM files 
 WHERE id = $1
 `
@@ -134,6 +140,8 @@ func (q *Queries) GetFileByUUID(ctx context.Context, id pgtype.UUID) (File, erro
 		&i.Mimetype,
 		&i.Size,
 		&i.Blake2,
+		&i.Score,
+		&i.Creator,
 		&i.Created,
 		&i.Updated,
 	)

server/internal/sqlc/queries-msoffice.sql.go

@@ -11,35 +11,8 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
-const getMSOfficeData = `-- name: GetMSOfficeData :one
-SELECT t1.file_id, t1.data AS oleid, t2.data AS olevba, t3.data AS mraptor
- FROM msoffice_oleid as t1
- LEFT join msoffice_olevba AS t2 ON t2.file_id = t1.file_id
- LEFT JOIN msoffice_mraptor AS t3 ON t3.file_id = t1.file_id
- WHERE t1.file_id = $1
-`
-
-type GetMSOfficeDataRow struct {
-	FileID  pgtype.UUID
-	Oleid   []byte
-	Olevba  []byte
-	Mraptor []byte
-}
-
-func (q *Queries) GetMSOfficeData(ctx context.Context, dollar_1 pgtype.UUID) (GetMSOfficeDataRow, error) {
-	row := q.db.QueryRow(ctx, getMSOfficeData, dollar_1)
-	var i GetMSOfficeDataRow
-	err := row.Scan(
-		&i.FileID,
-		&i.Oleid,
-		&i.Olevba,
-		&i.Mraptor,
-	)
-	return i, err
-}
-
 const getMSOfficeResults = `-- name: GetMSOfficeResults :one
-SELECT id, file_id, verdict, container_format, encrypted, file_format, vba_macros, xlm_macros, vba_stomping, nb_autoexec, nb_iocs, nb_macros, nb_suspicious, olevba_results, macros FROM msoffice
+SELECT id, file_id, verdict, container_format, encrypted, file_format, vba_macros, xlm_macros, vba_stomping, nb_autoexec, nb_iocs, nb_macros, nb_suspicious, olevba_results, macros, created FROM msoffice
 WHERE file_id = $1
 LIMIT 1
 `
@@ -63,58 +36,11 @@ func (q *Queries) GetMSOfficeResults(ctx context.Context, fileID pgtype.UUID) (M
 		&i.NbSuspicious,
 		&i.OlevbaResults,
 		&i.Macros,
+		&i.Created,
 	)
 	return i, err
 }
 
-const insertFileMsofficeMraptor = `-- name: InsertFileMsofficeMraptor :exec
-INSERT INTO msoffice_mraptor (
-  file_id, data
-) VALUES ($1, $2)
-`
-
-type InsertFileMsofficeMraptorParams struct {
-	Column1 pgtype.UUID
-	Column2 []byte
-}
-
-func (q *Queries) InsertFileMsofficeMraptor(ctx context.Context, arg InsertFileMsofficeMraptorParams) error {
-	_, err := q.db.Exec(ctx, insertFileMsofficeMraptor, arg.Column1, arg.Column2)
-	return err
-}
-
-const insertFileMsofficeOleid = `-- name: InsertFileMsofficeOleid :exec
-INSERT INTO msoffice_oleid (
-  file_id, data
-) VALUES ($1, $2)
-`
-
-type InsertFileMsofficeOleidParams struct {
-	Column1 pgtype.UUID
-	Column2 []byte
-}
-
-func (q *Queries) InsertFileMsofficeOleid(ctx context.Context, arg InsertFileMsofficeOleidParams) error {
-	_, err := q.db.Exec(ctx, insertFileMsofficeOleid, arg.Column1, arg.Column2)
-	return err
-}
-
-const insertFileMsofficeOlevba = `-- name: InsertFileMsofficeOlevba :exec
-INSERT INTO msoffice_olevba (
-  file_id, data
-) VALUES ($1, $2)
-`
-
-type InsertFileMsofficeOlevbaParams struct {
-	Column1 pgtype.UUID
-	Column2 []byte
-}
-
-func (q *Queries) InsertFileMsofficeOlevba(ctx context.Context, arg InsertFileMsofficeOlevbaParams) error {
-	_, err := q.db.Exec(ctx, insertFileMsofficeOlevba, arg.Column1, arg.Column2)
-	return err
-}
-
 const insertMSOfficeResults = `-- name: InsertMSOfficeResults :exec
 INSERT INTO msoffice (
   file_id, verdict, container_format, encrypted, file_format, vba_macros, xlm_macros,

server/internal/sqlc/queries-processing_jobs.sql.go

@@ -125,6 +125,7 @@ func (q *Queries) GetJob(ctx context.Context, id int64) (ProcessingJob, error) {
 const getJobsForFile = `-- name: GetJobsForFile :many
 SELECT id, file_id, created, started, completed, status, job_type, error, messages FROM processing_jobs
 WHERE file_id = $1
+ORDER BY created DESC
 `
 
 func (q *Queries) GetJobsForFile(ctx context.Context, fileID pgtype.UUID) ([]ProcessingJob, error) {

server/internal/sqlc/queries-yara.sql.go

@@ -22,7 +22,7 @@ func (q *Queries) DeleteYaraResults(ctx context.Context, id int64) error {
 }
 
 const getYaraResults = `-- name: GetYaraResults :one
-SELECT id, file_id, matched FROM yara_results
+SELECT id, file_id, matched, created FROM yara_results
 WHERE file_id = $1
 LIMIT 1
 `
@@ -30,7 +30,12 @@ LIMIT 1
 func (q *Queries) GetYaraResults(ctx context.Context, fileID pgtype.UUID) (YaraResult, error) {
 	row := q.db.QueryRow(ctx, getYaraResults, fileID)
 	var i YaraResult
-	err := row.Scan(&i.ID, &i.FileID, &i.Matched)
+	err := row.Scan(
+		&i.ID,
+		&i.FileID,
+		&i.Matched,
+		&i.Created,
+	)
 	return i, err
 }
 
@@ -38,7 +43,7 @@ const insertYaraResults = `-- name: InsertYaraResults :one
 INSERT INTO yara_results (
  file_id, matched
 ) VALUES ($1, $2)
-RETURNING id, file_id, matched
+RETURNING id, file_id, matched, created
 `
 
 type InsertYaraResultsParams struct {
@@ -49,6 +54,11 @@ type InsertYaraResultsParams struct {
 func (q *Queries) InsertYaraResults(ctx context.Context, arg InsertYaraResultsParams) (YaraResult, error) {
 	row := q.db.QueryRow(ctx, insertYaraResults, arg.FileID, arg.Matched)
 	var i YaraResult
-	err := row.Scan(&i.ID, &i.FileID, &i.Matched)
+	err := row.Scan(
+		&i.ID,
+		&i.FileID,
+		&i.Matched,
+		&i.Created,
+	)
 	return i, err
 }