jmb/scanfile
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sql stuff

Browse source
This commit is contained in:
Johannes Bülow 2025-08-15 19:16:20 +02:00
parent fed7862874
commit f8850aa89e
Signed by: jmb
GPG key ID: B56971CF7B8F83A6
7 changed files with 141 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
server/internal/database/schema.sql
View file

@ -2,6 +2,17 @@
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
-- Tables
CREATE TABLE IF NOT EXISTS users (
id BIGSERIAL PRIMARY KEY,
user_name TEXT NOT NULL UNIQUE,
display_name TEXT NOT NULL,
pw_hash BYTEA NOT NULL,
email_address TEXT NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
created TIMESTAMP DEFAULT NOW() NOT NULL
);
CREATE TABLE IF NOT EXISTS files (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
@ -9,6 +20,8 @@ CREATE TABLE IF NOT EXISTS files (
mimetype TEXT NOT NULL,
size BIGINT NOT NULL,
blake2 BYTEA NOT NULL UNIQUE,
score DECIMAL,
creator BIGINT REFERENCES users (id),
created TIMESTAMP DEFAULT NOW() NOT NULL,
updated TIMESTAMP DEFAULT NOW() NOT NULL
);
@ -16,7 +29,7 @@ CREATE TABLE IF NOT EXISTS files (
CREATE TABLE IF NOT EXISTS processing_jobs (
id BIGSERIAL PRIMARY KEY,
file_id UUID REFERENCES files (id) ON DELETE CASCADE,
created TIMESTAMP DEFAULT NOW(),
created TIMESTAMP DEFAULT NOW() NOT NULL,
started TIMESTAMP,
completed TIMESTAMP,
status TEXT,
@ -28,7 +41,8 @@ CREATE TABLE IF NOT EXISTS processing_jobs (
CREATE TABLE IF NOT EXISTS diec (
id BIGSERIAL PRIMARY KEY,
file_id UUID REFERENCES files (id) ON DELETE CASCADE,
data JSONB
data JSONB,
created TIMESTAMP DEFAULT NOW() NOT NULL
);
CREATE TABLE IF NOT EXISTS msoffice (
@ -46,7 +60,8 @@ CREATE TABLE IF NOT EXISTS msoffice (
nb_macros INTEGER,
nb_suspicious INTEGER,
olevba_results TEXT[][],
macros TEXT[][]
macros TEXT[][],
created TIMESTAMP DEFAULT NOW() NOT NULL
);
@ -57,23 +72,32 @@ CREATE TABLE IF NOT EXISTS file_properties (
md5 BYTEA,
libmagic_mime TEXT,
libmagic_extension TEXT,
libmagic_apple TEXT
libmagic_apple TEXT,
created TIMESTAMP DEFAULT NOW() NOT NULL
);
CREATE TABLE IF NOT EXISTS yara_results (
id BIGSERIAL PRIMARY KEY,
file_id UUID REFERENCES files (id) ON DELETE CASCADE,
matched TEXT[]
matched TEXT[],
created TIMESTAMP DEFAULT NOW() NOT NULL
);
CREATE TABLE IF NOT EXISTS capa_results (
id BIGSERIAL PRIMARY KEY,
file_id UUID REFERENCES files (id) ON DELETE CASCADE,
data JSONB,
type TEXT DEFAULT 'default' NOT NULL,
created TIMESTAMP DEFAULT NOW() NOT NULL
);
-- Indices
-- Since tables will be heavily accessed by file_id, there should be indices for them
CREATE INDEX idx_diec_file_id ON diec (file_id);
CREATE INDEX idx_processing_jobs_file_id ON processing_jobs (file_id);
CREATE INDEX idx_msoffice_oleid_file_id ON msoffice_oleid (file_id);
CREATE INDEX idx_msoffice_olevba_file_id ON msoffice_olevba (file_id);
CREATE INDEX idx_msoffice_mraptor_file_id ON msoffice_mraptor (file_id);
CREATE INDEX idx_msoffice_results_file_id ON msoffice (file_id);
CREATE INDEX idx_file_properties_file_id ON file_properties (file_id);
CREATE INDEX idx_file_id ON files (id);
CREATE INDEX idx_yara_results_file_id ON yara_results (file_id);
CREATE INDEX idx_user_name ON users (user_name);
CREATE INDEX idx_capa_results_file_id ON capa_results (file_id);

24
server/internal/sqlc/models.go
View file

@ -8,10 +8,19 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
type CapaResult struct {
ID int64
FileID pgtype.UUID
Data []byte
Type string
Created pgtype.Timestamp
}
type Diec struct {
ID int64
FileID pgtype.UUID
Data []byte
Created pgtype.Timestamp
}
type File struct {
@ -21,6 +30,8 @@ type File struct {
Mimetype string
Size int64
Blake2 []byte
Score pgtype.Numeric
Creator pgtype.Int8
Created pgtype.Timestamp
Updated pgtype.Timestamp
}
@ -33,6 +44,7 @@ type FileProperty struct {
LibmagicMime pgtype.Text
LibmagicExtension pgtype.Text
LibmagicApple pgtype.Text
Created pgtype.Timestamp
}
type Msoffice struct {
@ -51,6 +63,7 @@ type Msoffice struct {
NbSuspicious pgtype.Int4
OlevbaResults [][]string
Macros [][]string
Created pgtype.Timestamp
}
type ProcessingJob struct {
@ -65,8 +78,19 @@ type ProcessingJob struct {
Messages []string
}
type User struct {
ID int64
UserName string
DisplayName string
PwHash []byte
EmailAddress string
Enabled bool
Created pgtype.Timestamp
}
type YaraResult struct {
ID int64
FileID pgtype.UUID
Matched []string
Created pgtype.Timestamp
}

54
server/internal/sqlc/queries-file_properties.sql.go
View file

@ -11,8 +11,43 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const getFileCapa = `-- name: GetFileCapa :one
SELECT id, file_id, data, type, created FROM capa_results
WHERE file_id = $1
`
func (q *Queries) GetFileCapa(ctx context.Context, fileID pgtype.UUID) (CapaResult, error) {
row := q.db.QueryRow(ctx, getFileCapa, fileID)
var i CapaResult
err := row.Scan(
&i.ID,
&i.FileID,
&i.Data,
&i.Type,
&i.Created,
)
return i, err
}
const getFileDIEC = `-- name: GetFileDIEC :one
SELECT id, file_id, data, created FROM diec
WHERE file_id = $1
`
func (q *Queries) GetFileDIEC(ctx context.Context, fileID pgtype.UUID) (Diec, error) {
row := q.db.QueryRow(ctx, getFileDIEC, fileID)
var i Diec
err := row.Scan(
&i.ID,
&i.FileID,
&i.Data,
&i.Created,
)
return i, err
}
const getFileProperties = `-- name: GetFileProperties :one
SELECT id, file_id, sha256, md5, libmagic_mime, libmagic_extension, libmagic_apple FROM file_properties
SELECT id, file_id, sha256, md5, libmagic_mime, libmagic_extension, libmagic_apple, created FROM file_properties
WHERE file_id = $1
`
@ -27,10 +62,27 @@ func (q *Queries) GetFileProperties(ctx context.Context, fileID pgtype.UUID) (Fi
&i.LibmagicMime,
&i.LibmagicExtension,
&i.LibmagicApple,
&i.Created,
)
return i, err
}
const insertFileCapa = `-- name: InsertFileCapa :exec
INSERT INTO capa_results (
file_id, data
) VALUES ($1, $2)
`
type InsertFileCapaParams struct {
FileID pgtype.UUID
Data []byte
}
func (q *Queries) InsertFileCapa(ctx context.Context, arg InsertFileCapaParams) error {
_, err := q.db.Exec(ctx, insertFileCapa, arg.FileID, arg.Data)
return err
}
const insertFileDIEC = `-- name: InsertFileDIEC :exec
INSERT INTO diec (
file_id, data

16
server/internal/sqlc/queries-files.sql.go
View file

@ -18,7 +18,7 @@ INSERT INTO files (
$1,$2,$3,$4,$5
)
ON CONFLICT DO NOTHING -- Handle this in application code
RETURNING id, name, description, mimetype, size, blake2, created, updated
RETURNING id, name, description, mimetype, size, blake2, score, creator, created, updated
`
type CreateFileParams struct {
@ -45,6 +45,8 @@ func (q *Queries) CreateFile(ctx context.Context, arg CreateFileParams) (File, e
&i.Mimetype,
&i.Size,
&i.Blake2,
&i.Score,
&i.Creator,
&i.Created,
&i.Updated,
)
@ -63,7 +65,7 @@ func (q *Queries) DeleteFile(ctx context.Context, id pgtype.UUID) error {
}
const getAllFiles = `-- name: GetAllFiles :many
SELECT id, name, description, mimetype, size, blake2, created, updated FROM files
SELECT id, name, description, mimetype, size, blake2, score, creator, created, updated FROM files
ORDER BY created DESC
`
@ -83,6 +85,8 @@ func (q *Queries) GetAllFiles(ctx context.Context) ([]File, error) {
&i.Mimetype,
&i.Size,
&i.Blake2,
&i.Score,
&i.Creator,
&i.Created,
&i.Updated,
); err != nil {
@ -97,7 +101,7 @@ func (q *Queries) GetAllFiles(ctx context.Context) ([]File, error) {
}
const getFileByBlake2 = `-- name: GetFileByBlake2 :one
SELECT id, name, description, mimetype, size, blake2, created, updated
SELECT id, name, description, mimetype, size, blake2, score, creator, created, updated
FROM files
WHERE blake2 = $1
`
@ -112,6 +116,8 @@ func (q *Queries) GetFileByBlake2(ctx context.Context, blake2 []byte) (File, err
&i.Mimetype,
&i.Size,
&i.Blake2,
&i.Score,
&i.Creator,
&i.Created,
&i.Updated,
)
@ -119,7 +125,7 @@ func (q *Queries) GetFileByBlake2(ctx context.Context, blake2 []byte) (File, err
}
const getFileByUUID = `-- name: GetFileByUUID :one
SELECT id, name, description, mimetype, size, blake2, created, updated
SELECT id, name, description, mimetype, size, blake2, score, creator, created, updated
FROM files
WHERE id = $1
`
@ -134,6 +140,8 @@ func (q *Queries) GetFileByUUID(ctx context.Context, id pgtype.UUID) (File, erro
&i.Mimetype,
&i.Size,
&i.Blake2,
&i.Score,
&i.Creator,
&i.Created,
&i.Updated,
)

78
server/internal/sqlc/queries-msoffice.sql.go
View file

@ -11,35 +11,8 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const getMSOfficeData = `-- name: GetMSOfficeData :one
SELECT t1.file_id, t1.data AS oleid, t2.data AS olevba, t3.data AS mraptor
FROM msoffice_oleid as t1
LEFT join msoffice_olevba AS t2 ON t2.file_id = t1.file_id
LEFT JOIN msoffice_mraptor AS t3 ON t3.file_id = t1.file_id
WHERE t1.file_id = $1
`
type GetMSOfficeDataRow struct {
FileID pgtype.UUID
Oleid []byte
Olevba []byte
Mraptor []byte
}
func (q *Queries) GetMSOfficeData(ctx context.Context, dollar_1 pgtype.UUID) (GetMSOfficeDataRow, error) {
row := q.db.QueryRow(ctx, getMSOfficeData, dollar_1)
var i GetMSOfficeDataRow
err := row.Scan(
&i.FileID,
&i.Oleid,
&i.Olevba,
&i.Mraptor,
)
return i, err
}
const getMSOfficeResults = `-- name: GetMSOfficeResults :one
SELECT id, file_id, verdict, container_format, encrypted, file_format, vba_macros, xlm_macros, vba_stomping, nb_autoexec, nb_iocs, nb_macros, nb_suspicious, olevba_results, macros FROM msoffice
SELECT id, file_id, verdict, container_format, encrypted, file_format, vba_macros, xlm_macros, vba_stomping, nb_autoexec, nb_iocs, nb_macros, nb_suspicious, olevba_results, macros, created FROM msoffice
WHERE file_id = $1
LIMIT 1
`
@ -63,58 +36,11 @@ func (q *Queries) GetMSOfficeResults(ctx context.Context, fileID pgtype.UUID) (M
&i.NbSuspicious,
&i.OlevbaResults,
&i.Macros,
&i.Created,
)
return i, err
}
const insertFileMsofficeMraptor = `-- name: InsertFileMsofficeMraptor :exec
INSERT INTO msoffice_mraptor (
file_id, data
) VALUES ($1, $2)
`
type InsertFileMsofficeMraptorParams struct {
Column1 pgtype.UUID
Column2 []byte
}
func (q *Queries) InsertFileMsofficeMraptor(ctx context.Context, arg InsertFileMsofficeMraptorParams) error {
_, err := q.db.Exec(ctx, insertFileMsofficeMraptor, arg.Column1, arg.Column2)
return err
}
const insertFileMsofficeOleid = `-- name: InsertFileMsofficeOleid :exec
INSERT INTO msoffice_oleid (
file_id, data
) VALUES ($1, $2)
`
type InsertFileMsofficeOleidParams struct {
Column1 pgtype.UUID
Column2 []byte
}
func (q *Queries) InsertFileMsofficeOleid(ctx context.Context, arg InsertFileMsofficeOleidParams) error {
_, err := q.db.Exec(ctx, insertFileMsofficeOleid, arg.Column1, arg.Column2)
return err
}
const insertFileMsofficeOlevba = `-- name: InsertFileMsofficeOlevba :exec
INSERT INTO msoffice_olevba (
file_id, data
) VALUES ($1, $2)
`
type InsertFileMsofficeOlevbaParams struct {
Column1 pgtype.UUID
Column2 []byte
}
func (q *Queries) InsertFileMsofficeOlevba(ctx context.Context, arg InsertFileMsofficeOlevbaParams) error {
_, err := q.db.Exec(ctx, insertFileMsofficeOlevba, arg.Column1, arg.Column2)
return err
}
const insertMSOfficeResults = `-- name: InsertMSOfficeResults :exec
INSERT INTO msoffice (
file_id, verdict, container_format, encrypted, file_format, vba_macros, xlm_macros,

1
server/internal/sqlc/queries-processing_jobs.sql.go
View file

@ -125,6 +125,7 @@ func (q *Queries) GetJob(ctx context.Context, id int64) (ProcessingJob, error) {
const getJobsForFile = `-- name: GetJobsForFile :many
SELECT id, file_id, created, started, completed, status, job_type, error, messages FROM processing_jobs
WHERE file_id = $1
ORDER BY created DESC
`
func (q *Queries) GetJobsForFile(ctx context.Context, fileID pgtype.UUID) ([]ProcessingJob, error) {

18
server/internal/sqlc/queries-yara.sql.go
View file

@ -22,7 +22,7 @@ func (q *Queries) DeleteYaraResults(ctx context.Context, id int64) error {
}
const getYaraResults = `-- name: GetYaraResults :one
SELECT id, file_id, matched FROM yara_results
SELECT id, file_id, matched, created FROM yara_results
WHERE file_id = $1
LIMIT 1
`
@ -30,7 +30,12 @@ LIMIT 1
func (q *Queries) GetYaraResults(ctx context.Context, fileID pgtype.UUID) (YaraResult, error) {
row := q.db.QueryRow(ctx, getYaraResults, fileID)
var i YaraResult
err := row.Scan(&i.ID, &i.FileID, &i.Matched)
err := row.Scan(
&i.ID,
&i.FileID,
&i.Matched,
&i.Created,
)
return i, err
}
@ -38,7 +43,7 @@ const insertYaraResults = `-- name: InsertYaraResults :one
INSERT INTO yara_results (
file_id, matched
) VALUES ($1, $2)
RETURNING id, file_id, matched
RETURNING id, file_id, matched, created
`
type InsertYaraResultsParams struct {
@ -49,6 +54,11 @@ type InsertYaraResultsParams struct {
func (q *Queries) InsertYaraResults(ctx context.Context, arg InsertYaraResultsParams) (YaraResult, error) {
row := q.db.QueryRow(ctx, insertYaraResults, arg.FileID, arg.Matched)
var i YaraResult
err := row.Scan(&i.ID, &i.FileID, &i.Matched)
err := row.Scan(
&i.ID,
&i.FileID,
&i.Matched,
&i.Created,
)
return i, err
}