diff --git a/server/internal/database/schema.sql b/server/internal/database/schema.sql index f0bb2c8..62c7f99 100644 --- a/server/internal/database/schema.sql +++ b/server/internal/database/schema.sql @@ -2,6 +2,17 @@ CREATE EXTENSION IF NOT EXISTS "pgcrypto"; -- Tables + +CREATE TABLE IF NOT EXISTS users ( + id BIGSERIAL PRIMARY KEY, + user_name TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + pw_hash BYTEA NOT NULL, + email_address TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + created TIMESTAMP DEFAULT NOW() NOT NULL +); + CREATE TABLE IF NOT EXISTS files ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), name TEXT NOT NULL, @@ -9,6 +20,8 @@ CREATE TABLE IF NOT EXISTS files ( mimetype TEXT NOT NULL, size BIGINT NOT NULL, blake2 BYTEA NOT NULL UNIQUE, + score DECIMAL, + creator BIGINT REFERENCES users (id), created TIMESTAMP DEFAULT NOW() NOT NULL, updated TIMESTAMP DEFAULT NOW() NOT NULL ); @@ -16,7 +29,7 @@ CREATE TABLE IF NOT EXISTS files ( CREATE TABLE IF NOT EXISTS processing_jobs ( id BIGSERIAL PRIMARY KEY, file_id UUID REFERENCES files (id) ON DELETE CASCADE, - created TIMESTAMP DEFAULT NOW(), + created TIMESTAMP DEFAULT NOW() NOT NULL, started TIMESTAMP, completed TIMESTAMP, status TEXT, @@ -28,7 +41,8 @@ CREATE TABLE IF NOT EXISTS processing_jobs ( CREATE TABLE IF NOT EXISTS diec ( id BIGSERIAL PRIMARY KEY, file_id UUID REFERENCES files (id) ON DELETE CASCADE, - data JSONB + data JSONB, + created TIMESTAMP DEFAULT NOW() NOT NULL ); CREATE TABLE IF NOT EXISTS msoffice ( @@ -46,7 +60,8 @@ CREATE TABLE IF NOT EXISTS msoffice ( nb_macros INTEGER, nb_suspicious INTEGER, olevba_results TEXT[][], - macros TEXT[][] + macros TEXT[][], + created TIMESTAMP DEFAULT NOW() NOT NULL ); @@ -57,23 +72,32 @@ CREATE TABLE IF NOT EXISTS file_properties ( md5 BYTEA, libmagic_mime TEXT, libmagic_extension TEXT, - libmagic_apple TEXT + libmagic_apple TEXT, + created TIMESTAMP DEFAULT NOW() NOT NULL ); CREATE TABLE IF NOT EXISTS yara_results ( id BIGSERIAL PRIMARY KEY, file_id UUID REFERENCES files (id) ON DELETE CASCADE, - matched TEXT[] + matched TEXT[], + created TIMESTAMP DEFAULT NOW() NOT NULL +); + +CREATE TABLE IF NOT EXISTS capa_results ( + id BIGSERIAL PRIMARY KEY, + file_id UUID REFERENCES files (id) ON DELETE CASCADE, + data JSONB, + type TEXT DEFAULT 'default' NOT NULL, + created TIMESTAMP DEFAULT NOW() NOT NULL ); -- Indices -- Since tables will be heavily accessed by file_id, there should be indices for them CREATE INDEX idx_diec_file_id ON diec (file_id); CREATE INDEX idx_processing_jobs_file_id ON processing_jobs (file_id); -CREATE INDEX idx_msoffice_oleid_file_id ON msoffice_oleid (file_id); -CREATE INDEX idx_msoffice_olevba_file_id ON msoffice_olevba (file_id); -CREATE INDEX idx_msoffice_mraptor_file_id ON msoffice_mraptor (file_id); CREATE INDEX idx_msoffice_results_file_id ON msoffice (file_id); CREATE INDEX idx_file_properties_file_id ON file_properties (file_id); CREATE INDEX idx_file_id ON files (id); CREATE INDEX idx_yara_results_file_id ON yara_results (file_id); +CREATE INDEX idx_user_name ON users (user_name); +CREATE INDEX idx_capa_results_file_id ON capa_results (file_id); diff --git a/server/internal/sqlc/models.go b/server/internal/sqlc/models.go index 0239a18..9e5c09d 100644 --- a/server/internal/sqlc/models.go +++ b/server/internal/sqlc/models.go @@ -8,10 +8,19 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +type CapaResult struct { + ID int64 + FileID pgtype.UUID + Data []byte + Type string + Created pgtype.Timestamp +} + type Diec struct { - ID int64 - FileID pgtype.UUID - Data []byte + ID int64 + FileID pgtype.UUID + Data []byte + Created pgtype.Timestamp } type File struct { @@ -21,6 +30,8 @@ type File struct { Mimetype string Size int64 Blake2 []byte + Score pgtype.Numeric + Creator pgtype.Int8 Created pgtype.Timestamp Updated pgtype.Timestamp } @@ -33,6 +44,7 @@ type FileProperty struct { LibmagicMime pgtype.Text LibmagicExtension pgtype.Text LibmagicApple pgtype.Text + Created pgtype.Timestamp } type Msoffice struct { @@ -51,6 +63,7 @@ type Msoffice struct { NbSuspicious pgtype.Int4 OlevbaResults [][]string Macros [][]string + Created pgtype.Timestamp } type ProcessingJob struct { @@ -65,8 +78,19 @@ type ProcessingJob struct { Messages []string } +type User struct { + ID int64 + UserName string + DisplayName string + PwHash []byte + EmailAddress string + Enabled bool + Created pgtype.Timestamp +} + type YaraResult struct { ID int64 FileID pgtype.UUID Matched []string + Created pgtype.Timestamp } diff --git a/server/internal/sqlc/queries-file_properties.sql.go b/server/internal/sqlc/queries-file_properties.sql.go index c36ecde..e88f8f4 100644 --- a/server/internal/sqlc/queries-file_properties.sql.go +++ b/server/internal/sqlc/queries-file_properties.sql.go @@ -11,8 +11,43 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const getFileCapa = `-- name: GetFileCapa :one +SELECT id, file_id, data, type, created FROM capa_results +WHERE file_id = $1 +` + +func (q *Queries) GetFileCapa(ctx context.Context, fileID pgtype.UUID) (CapaResult, error) { + row := q.db.QueryRow(ctx, getFileCapa, fileID) + var i CapaResult + err := row.Scan( + &i.ID, + &i.FileID, + &i.Data, + &i.Type, + &i.Created, + ) + return i, err +} + +const getFileDIEC = `-- name: GetFileDIEC :one +SELECT id, file_id, data, created FROM diec +WHERE file_id = $1 +` + +func (q *Queries) GetFileDIEC(ctx context.Context, fileID pgtype.UUID) (Diec, error) { + row := q.db.QueryRow(ctx, getFileDIEC, fileID) + var i Diec + err := row.Scan( + &i.ID, + &i.FileID, + &i.Data, + &i.Created, + ) + return i, err +} + const getFileProperties = `-- name: GetFileProperties :one -SELECT id, file_id, sha256, md5, libmagic_mime, libmagic_extension, libmagic_apple FROM file_properties +SELECT id, file_id, sha256, md5, libmagic_mime, libmagic_extension, libmagic_apple, created FROM file_properties WHERE file_id = $1 ` @@ -27,10 +62,27 @@ func (q *Queries) GetFileProperties(ctx context.Context, fileID pgtype.UUID) (Fi &i.LibmagicMime, &i.LibmagicExtension, &i.LibmagicApple, + &i.Created, ) return i, err } +const insertFileCapa = `-- name: InsertFileCapa :exec +INSERT INTO capa_results ( + file_id, data +) VALUES ($1, $2) +` + +type InsertFileCapaParams struct { + FileID pgtype.UUID + Data []byte +} + +func (q *Queries) InsertFileCapa(ctx context.Context, arg InsertFileCapaParams) error { + _, err := q.db.Exec(ctx, insertFileCapa, arg.FileID, arg.Data) + return err +} + const insertFileDIEC = `-- name: InsertFileDIEC :exec INSERT INTO diec ( file_id, data diff --git a/server/internal/sqlc/queries-files.sql.go b/server/internal/sqlc/queries-files.sql.go index a46aa64..dd3c374 100644 --- a/server/internal/sqlc/queries-files.sql.go +++ b/server/internal/sqlc/queries-files.sql.go @@ -18,7 +18,7 @@ INSERT INTO files ( $1,$2,$3,$4,$5 ) ON CONFLICT DO NOTHING -- Handle this in application code -RETURNING id, name, description, mimetype, size, blake2, created, updated +RETURNING id, name, description, mimetype, size, blake2, score, creator, created, updated ` type CreateFileParams struct { @@ -45,6 +45,8 @@ func (q *Queries) CreateFile(ctx context.Context, arg CreateFileParams) (File, e &i.Mimetype, &i.Size, &i.Blake2, + &i.Score, + &i.Creator, &i.Created, &i.Updated, ) @@ -63,7 +65,7 @@ func (q *Queries) DeleteFile(ctx context.Context, id pgtype.UUID) error { } const getAllFiles = `-- name: GetAllFiles :many -SELECT id, name, description, mimetype, size, blake2, created, updated FROM files +SELECT id, name, description, mimetype, size, blake2, score, creator, created, updated FROM files ORDER BY created DESC ` @@ -83,6 +85,8 @@ func (q *Queries) GetAllFiles(ctx context.Context) ([]File, error) { &i.Mimetype, &i.Size, &i.Blake2, + &i.Score, + &i.Creator, &i.Created, &i.Updated, ); err != nil { @@ -97,7 +101,7 @@ func (q *Queries) GetAllFiles(ctx context.Context) ([]File, error) { } const getFileByBlake2 = `-- name: GetFileByBlake2 :one -SELECT id, name, description, mimetype, size, blake2, created, updated +SELECT id, name, description, mimetype, size, blake2, score, creator, created, updated FROM files WHERE blake2 = $1 ` @@ -112,6 +116,8 @@ func (q *Queries) GetFileByBlake2(ctx context.Context, blake2 []byte) (File, err &i.Mimetype, &i.Size, &i.Blake2, + &i.Score, + &i.Creator, &i.Created, &i.Updated, ) @@ -119,7 +125,7 @@ func (q *Queries) GetFileByBlake2(ctx context.Context, blake2 []byte) (File, err } const getFileByUUID = `-- name: GetFileByUUID :one -SELECT id, name, description, mimetype, size, blake2, created, updated +SELECT id, name, description, mimetype, size, blake2, score, creator, created, updated FROM files WHERE id = $1 ` @@ -134,6 +140,8 @@ func (q *Queries) GetFileByUUID(ctx context.Context, id pgtype.UUID) (File, erro &i.Mimetype, &i.Size, &i.Blake2, + &i.Score, + &i.Creator, &i.Created, &i.Updated, ) diff --git a/server/internal/sqlc/queries-msoffice.sql.go b/server/internal/sqlc/queries-msoffice.sql.go index 07a7718..3bedebe 100644 --- a/server/internal/sqlc/queries-msoffice.sql.go +++ b/server/internal/sqlc/queries-msoffice.sql.go @@ -11,35 +11,8 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -const getMSOfficeData = `-- name: GetMSOfficeData :one -SELECT t1.file_id, t1.data AS oleid, t2.data AS olevba, t3.data AS mraptor - FROM msoffice_oleid as t1 - LEFT join msoffice_olevba AS t2 ON t2.file_id = t1.file_id - LEFT JOIN msoffice_mraptor AS t3 ON t3.file_id = t1.file_id - WHERE t1.file_id = $1 -` - -type GetMSOfficeDataRow struct { - FileID pgtype.UUID - Oleid []byte - Olevba []byte - Mraptor []byte -} - -func (q *Queries) GetMSOfficeData(ctx context.Context, dollar_1 pgtype.UUID) (GetMSOfficeDataRow, error) { - row := q.db.QueryRow(ctx, getMSOfficeData, dollar_1) - var i GetMSOfficeDataRow - err := row.Scan( - &i.FileID, - &i.Oleid, - &i.Olevba, - &i.Mraptor, - ) - return i, err -} - const getMSOfficeResults = `-- name: GetMSOfficeResults :one -SELECT id, file_id, verdict, container_format, encrypted, file_format, vba_macros, xlm_macros, vba_stomping, nb_autoexec, nb_iocs, nb_macros, nb_suspicious, olevba_results, macros FROM msoffice +SELECT id, file_id, verdict, container_format, encrypted, file_format, vba_macros, xlm_macros, vba_stomping, nb_autoexec, nb_iocs, nb_macros, nb_suspicious, olevba_results, macros, created FROM msoffice WHERE file_id = $1 LIMIT 1 ` @@ -63,58 +36,11 @@ func (q *Queries) GetMSOfficeResults(ctx context.Context, fileID pgtype.UUID) (M &i.NbSuspicious, &i.OlevbaResults, &i.Macros, + &i.Created, ) return i, err } -const insertFileMsofficeMraptor = `-- name: InsertFileMsofficeMraptor :exec -INSERT INTO msoffice_mraptor ( - file_id, data -) VALUES ($1, $2) -` - -type InsertFileMsofficeMraptorParams struct { - Column1 pgtype.UUID - Column2 []byte -} - -func (q *Queries) InsertFileMsofficeMraptor(ctx context.Context, arg InsertFileMsofficeMraptorParams) error { - _, err := q.db.Exec(ctx, insertFileMsofficeMraptor, arg.Column1, arg.Column2) - return err -} - -const insertFileMsofficeOleid = `-- name: InsertFileMsofficeOleid :exec -INSERT INTO msoffice_oleid ( - file_id, data -) VALUES ($1, $2) -` - -type InsertFileMsofficeOleidParams struct { - Column1 pgtype.UUID - Column2 []byte -} - -func (q *Queries) InsertFileMsofficeOleid(ctx context.Context, arg InsertFileMsofficeOleidParams) error { - _, err := q.db.Exec(ctx, insertFileMsofficeOleid, arg.Column1, arg.Column2) - return err -} - -const insertFileMsofficeOlevba = `-- name: InsertFileMsofficeOlevba :exec -INSERT INTO msoffice_olevba ( - file_id, data -) VALUES ($1, $2) -` - -type InsertFileMsofficeOlevbaParams struct { - Column1 pgtype.UUID - Column2 []byte -} - -func (q *Queries) InsertFileMsofficeOlevba(ctx context.Context, arg InsertFileMsofficeOlevbaParams) error { - _, err := q.db.Exec(ctx, insertFileMsofficeOlevba, arg.Column1, arg.Column2) - return err -} - const insertMSOfficeResults = `-- name: InsertMSOfficeResults :exec INSERT INTO msoffice ( file_id, verdict, container_format, encrypted, file_format, vba_macros, xlm_macros, diff --git a/server/internal/sqlc/queries-processing_jobs.sql.go b/server/internal/sqlc/queries-processing_jobs.sql.go index 082e10b..dad453a 100644 --- a/server/internal/sqlc/queries-processing_jobs.sql.go +++ b/server/internal/sqlc/queries-processing_jobs.sql.go @@ -125,6 +125,7 @@ func (q *Queries) GetJob(ctx context.Context, id int64) (ProcessingJob, error) { const getJobsForFile = `-- name: GetJobsForFile :many SELECT id, file_id, created, started, completed, status, job_type, error, messages FROM processing_jobs WHERE file_id = $1 +ORDER BY created DESC ` func (q *Queries) GetJobsForFile(ctx context.Context, fileID pgtype.UUID) ([]ProcessingJob, error) { diff --git a/server/internal/sqlc/queries-yara.sql.go b/server/internal/sqlc/queries-yara.sql.go index 471df43..d3dc6ee 100644 --- a/server/internal/sqlc/queries-yara.sql.go +++ b/server/internal/sqlc/queries-yara.sql.go @@ -22,7 +22,7 @@ func (q *Queries) DeleteYaraResults(ctx context.Context, id int64) error { } const getYaraResults = `-- name: GetYaraResults :one -SELECT id, file_id, matched FROM yara_results +SELECT id, file_id, matched, created FROM yara_results WHERE file_id = $1 LIMIT 1 ` @@ -30,7 +30,12 @@ LIMIT 1 func (q *Queries) GetYaraResults(ctx context.Context, fileID pgtype.UUID) (YaraResult, error) { row := q.db.QueryRow(ctx, getYaraResults, fileID) var i YaraResult - err := row.Scan(&i.ID, &i.FileID, &i.Matched) + err := row.Scan( + &i.ID, + &i.FileID, + &i.Matched, + &i.Created, + ) return i, err } @@ -38,7 +43,7 @@ const insertYaraResults = `-- name: InsertYaraResults :one INSERT INTO yara_results ( file_id, matched ) VALUES ($1, $2) -RETURNING id, file_id, matched +RETURNING id, file_id, matched, created ` type InsertYaraResultsParams struct { @@ -49,6 +54,11 @@ type InsertYaraResultsParams struct { func (q *Queries) InsertYaraResults(ctx context.Context, arg InsertYaraResultsParams) (YaraResult, error) { row := q.db.QueryRow(ctx, insertYaraResults, arg.FileID, arg.Matched) var i YaraResult - err := row.Scan(&i.ID, &i.FileID, &i.Matched) + err := row.Scan( + &i.ID, + &i.FileID, + &i.Matched, + &i.Created, + ) return i, err }