worked more on database stuff, basics for file processing laid out

config.toml

@@ -17,4 +17,4 @@ debug = true
 path = "./storage/files/"
 
 [processing]
-executors = 5
+maxparallel = 5

go.mod

@@ -14,21 +14,14 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/spf13/viper v1.20.1
 	golang.org/x/crypto v0.38.0
-	xorm.io/xorm v1.3.9
 )
 
 require (
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
-	github.com/goccy/go-json v0.10.5 // indirect
-	github.com/golang/snappy v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/mattn/go-sqlite3 v1.14.27 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/sagikazarmark/locafero v0.9.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
@@ -36,10 +29,9 @@ require (
 	github.com/spf13/cast v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/syndtr/goleveldb v1.0.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/sync v0.14.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/text v0.25.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	xorm.io/builder v0.3.13 // indirect
 )

go.sum

@@ -1,7 +1,3 @@
-filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
-filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
-gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=
-gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=
 github.com/Oudwins/tailwind-merge-go v0.2.1 h1:jxRaEqGtwwwF48UuFIQ8g8XT7YSualNuGzCvQ89nPFE=
 github.com/Oudwins/tailwind-merge-go v0.2.1/go.mod h1:kkZodgOPvZQ8f7SIrlWkG/w1g9JTbtnptnePIh3V72U=
 github.com/a-h/templ v0.3.865 h1:nYn5EWm9EiXaDgWcMQaKiKvrydqgxDUtT1+4zU2C43A=
@@ -9,26 +5,14 @@ github.com/a-h/templ v0.3.865/go.mod h1:oLBbZVQ6//Q6zpvSMPTuBK0F3qOtBdFBcGRspcT+
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
-github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
 github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
-github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
-github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
-github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -39,8 +23,6 @@ github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzq
 github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik=
 github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
 github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -51,36 +33,16 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
-github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
-github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-sqlite3 v1.14.27 h1:drZCnuvf37yPfs95E5jd9s3XhdVWLal+6BOK6qrv6IU=
-github.com/mattn/go-sqlite3 v1.14.27/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
-github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
-github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k=
@@ -102,61 +64,19 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
-github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
-github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
 golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
-golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
-golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
-golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
 golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
 golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
 golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
-golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU=
-golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
-lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
-modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
-modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
-modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
-modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
-modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0=
-modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug=
-modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
-modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk=
-modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
-modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
-modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sqlite v1.20.4 h1:J8+m2trkN+KKoE7jglyHYYYiaq5xmz2HoHJIiBlRzbE=
-modernc.org/sqlite v1.20.4/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A=
-modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
-modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
-modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
-modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-xorm.io/builder v0.3.13 h1:a3jmiVVL19psGeXx8GIurTp7p0IIgqeDmwhcR6BAOAo=
-xorm.io/builder v0.3.13/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE=
-xorm.io/xorm v1.3.9 h1:TUovzS0ko+IQ1XnNLfs5dqK1cJl1H5uHpWbWqAQ04nU=
-xorm.io/xorm v1.3.9/go.mod h1:LsCCffeeYp63ssk0pKumP6l96WZcHix7ChpurcLNuMw=

server/internal/database/createFile.go

@@ -1,27 +1,33 @@
 package database
 
-
 import (
+	"context"
 	"encoding/hex"
 	"log/slog"
 
 	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5"
 	"golang.org/x/crypto/blake2b"
 
+	"git.jmbit.de/jmb/scanfile/server/internal/sqlc"
 	"git.jmbit.de/jmb/scanfile/server/internal/store"
 )
 
-
 // CreateFile() creates the filesystem object and the DB entry for a file
-func CreateFile(name string, fileBytes []byte, size int64) (File, error) {
+func CreateFile(ctx context.Context, name string, fileBytes []byte, size int64) (File, error) {
-	file := File{
+	file := sqlc.File{
 		Name: name,
 		Size: size,
 	}
 
   bl2hash := blake2b.Sum256(fileBytes)
-  file.Blake2 = hex.EncodeToString(bl2hash[:])
+  file.Blake2 = bl2hash[:]
-  fileExists, err := FileAlreadyExists(file.Blake2)
+  queries := sqlc.New(pool)
+  fileExists, err := queries.GetFileByBlake2()
+  if err == pgx.ErrNoRows {
+    slog.Info("File already exists", "file-uuid", )
+  }
+
   if err != nil {
     slog.Error("Could not check if a duplicate exists", "error", err)
     err = nil

server/internal/database/db.go

@@ -1,22 +1,22 @@
 package database
 
 import (
+	"context"
 	"fmt"
 	"log/slog"
 	"net/url"
 
+	"github.com/jackc/pgx/v5/pgxpool"
 	_ "github.com/lib/pq"
 	"github.com/spf13/viper"
-	"xorm.io/xorm"
-	"xorm.io/xorm/log"
-	"xorm.io/xorm/names"
 )
 
-var engine *xorm.Engine
 
-func Connect() (*xorm.Engine, error) {
+var pool *pgxpool.Pool
-	if engine != nil {
+
-		return engine, nil
+func Connect() (*pgxpool.Pool, error) {
+	if pool != nil {
+		return pool, nil
 	}
 	dburl := url.URL{
 		Scheme: "postgres",
@@ -24,27 +24,19 @@ func Connect() (*xorm.Engine, error) {
 		Host:   fmt.Sprintf("%s:%d", viper.GetString("db.host"), viper.GetInt("db.port")),
 		Path:   fmt.Sprintf("/%s", viper.GetString("db.database")),
 	}
-	engi, err := xorm.NewEngine("postgres", dburl.String())
+  conn, err := pgxpool.New(context.Background(), dburl.String())
 	if err != nil {
 		slog.Error("Could not connect to DB", "url", dburl.Redacted(), "error", err)
 		return nil, err
 	}
-	engi.SetMapper(names.LintGonicMapper)
-	err = engi.Sync(new(User), new(File))
-	if err != nil {
-		slog.Error("Error syncing DB schema", "error", err)
-	}
-  if viper.GetBool("db.debug") {
-    engi.SetLogLevel(log.LOG_DEBUG)
-    engi.ShowSQL(true)
-  }
-	engine = engi
 
-	return engi, nil
+  pool = conn
+
+	return conn, nil
 }
 
 func Ping() error {
-  err := engine.DB().Ping()
+  err := pool.Ping(context.Background())
   if err != nil {
     slog.Error("Error connecting to DB", "error", err)
     return err

server/internal/database/queries-processing_jobs.sql

@@ -15,3 +15,12 @@ UPDATE processing_jobs
 SET completed = NOW(),
   status = "completed"
 WHERE id = $1;
+
+-- name: GetJobsForFile :many
+SELECT * FROM processing_jobs
+WHERE file_id = $1;
+
+-- name: GetJob :one
+SELECT * FROM processing_jobs
+WHERE id = $1
+LIMIT 1;

server/internal/database/schema.sql

@@ -1,21 +1,66 @@
+-- Extensions
 CREATE EXTENSION IF NOT EXISTS "pgcrypto";
+
+-- Tables
 CREATE TABLE IF NOT EXISTS files (
   id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
   name TEXT NOT NULL,
   description TEXT,
   mimetype TEXT NOT NULL,
-  size INTEGER NOT NULL,
+  size BIGINT NOT NULL,
   blake2 BYTEA NOT NULL UNIQUE,
   created TIMESTAMP DEFAULT NOW() NOT NULL,
   updated TIMESTAMP DEFAULT NOW() NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS processing_jobs (
-  id SERIAL PRIMARY KEY,
+  id BIGSERIAL PRIMARY KEY,
-  file_id UUID REFERENCES files (id),
+  file_id UUID REFERENCES files (id) ON DELETE CASCADE,
   created TIMESTAMP DEFAULT NOW(),
   started TIMESTAMP,
   completed TIMESTAMP,
   status TEXT,
-  job_type TEXT
+  job_type TEXT,
+  error TEXT,
+  messages JSONB
 );
+
+CREATE TABLE IF NOT EXISTS diec (
+  id BIGSERIAL PRIMARY KEY,
+  file_id UUID REFERENCES files (id) ON DELETE CASCADE,
+  data JSONB,
+  created TIMESTAMP DEFAULT NOW() NOT NULL,
+  updated TIMESTAMP DEFAULT NOW() NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS msoffice_oleid (
+  id BIGSERIAL PRIMARY KEY,
+  file_id UUID REFERENCES files (id) ON DELETE CASCADE,
+  data JSONB,
+  created TIMESTAMP DEFAULT NOW() NOT NULL,
+  updated TIMESTAMP DEFAULT NOW() NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS msoffice_olevba (
+  id BIGSERIAL PRIMARY KEY,
+  file_id UUID REFERENCES files (id) ON DELETE CASCADE,
+  data JSONB,
+  created TIMESTAMP DEFAULT NOW() NOT NULL,
+  updated TIMESTAMP DEFAULT NOW() NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS msoffice_mraptor (
+  id BIGSERIAL PRIMARY KEY,
+  file_id UUID REFERENCES files (id) ON DELETE CASCADE,
+  data JSONB,
+  created TIMESTAMP DEFAULT NOW() NOT NULL,
+  updated TIMESTAMP DEFAULT NOW() NOT NULL
+);
+
+-- Indices
+-- Since tables will be heavily accessed by file_id, there should be indices for them
+CREATE INDEX idx_diec_file_id ON diec (file_id);
+CREATE INDEX idx_processing_jobs_file_id ON processing_jobs (file_id);
+CREATE INDEX idx_msoffice_oleid_file_id ON msoffice_oleid (file_id);
+CREATE INDEX idx_msoffice_olevba_file_id ON msoffice_olevba (file_id);
+CREATE INDEX idx_msoffice_mraptor_file_id ON msoffice_mraptor (file_id);

server/internal/processing/processing.go

@@ -1,4 +1,21 @@
 package processing
 
+import (
+	"sync"
+	"time"
+
+	"github.com/spf13/viper"
+)
+
+var semaphore chan struct{}
+var swg *sync.WaitGroup
+//Used to determine if a task was started by a previous instance that stalled out or died
+var startup time.Time
+
+func Setup(wg *sync.WaitGroup) {
+  semaphore = make(chan struct{}, viper.GetInt("processing.maxparallel"))
+  startup = time.Now()
+}
+
 func Submit()
 

server/internal/processing/types.go

@@ -0,0 +1,14 @@
+package processing
+
+// Microsoft Office Document
+const TypeMSOffice = "MSOffice"
+// Microsoft Windows Portable Executable
+const TypePE = "PE"
+// Linux/UNIX Executable Linkable Format
+const TypeELF = "ELF"
+// Java Archive (JAR)
+const TypeJAR = "JAR"
+// Archives (compressed etc.)
+const TypeArchive = "Archive"
+// Anything not implemented (yet)
+const TypeOther = "Other"

server/internal/sqlc/models.go

@@ -13,7 +13,7 @@ type File struct {
 	Name        string
 	Description pgtype.Text
 	Mimetype    string
-	Size        int32
+	Size        int64
 	Blake2      []byte
 	Created     pgtype.Timestamp
 	Updated     pgtype.Timestamp
@@ -27,4 +27,6 @@ type ProcessingJob struct {
 	Completed pgtype.Timestamp
 	Status    pgtype.Text
 	JobType   pgtype.Text
+	Error     pgtype.Text
+	Messages  []byte
 }

server/internal/sqlc/queries-files.sql.go

@@ -25,7 +25,7 @@ type CreateFileParams struct {
 	Name        string
 	Description pgtype.Text
 	Mimetype    string
-	Size        int32
+	Size        int64
 	Blake2      []byte
 }
 

server/internal/sqlc/queries-processing_jobs.sql.go

@@ -15,7 +15,7 @@ const createProcessingJob = `-- name: CreateProcessingJob :one
 INSERT INTO processing_jobs (
   file_id, job_type
 ) VALUES ($1,$2 )
-RETURNING id, file_id, created, started, completed, status, job_type
+RETURNING id, file_id, created, started, completed, status, job_type, error, messages
 `
 
 type CreateProcessingJobParams struct {
@@ -34,6 +34,8 @@ func (q *Queries) CreateProcessingJob(ctx context.Context, arg CreateProcessingJ
 		&i.Completed,
 		&i.Status,
 		&i.JobType,
+		&i.Error,
+		&i.Messages,
 	)
 	return i, err
 }
@@ -50,6 +52,64 @@ func (q *Queries) FinishProcessingJob(ctx context.Context, id int32) error {
 	return err
 }
 
+const getJob = `-- name: GetJob :one
+SELECT id, file_id, created, started, completed, status, job_type, error, messages FROM processing_jobs
+WHERE id = $1
+LIMIT 1
+`
+
+func (q *Queries) GetJob(ctx context.Context, id int32) (ProcessingJob, error) {
+	row := q.db.QueryRow(ctx, getJob, id)
+	var i ProcessingJob
+	err := row.Scan(
+		&i.ID,
+		&i.FileID,
+		&i.Created,
+		&i.Started,
+		&i.Completed,
+		&i.Status,
+		&i.JobType,
+		&i.Error,
+		&i.Messages,
+	)
+	return i, err
+}
+
+const getJobsForFile = `-- name: GetJobsForFile :many
+SELECT id, file_id, created, started, completed, status, job_type, error, messages FROM processing_jobs
+WHERE file_id = $1
+`
+
+func (q *Queries) GetJobsForFile(ctx context.Context, fileID pgtype.UUID) ([]ProcessingJob, error) {
+	rows, err := q.db.Query(ctx, getJobsForFile, fileID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ProcessingJob
+	for rows.Next() {
+		var i ProcessingJob
+		if err := rows.Scan(
+			&i.ID,
+			&i.FileID,
+			&i.Created,
+			&i.Started,
+			&i.Completed,
+			&i.Status,
+			&i.JobType,
+			&i.Error,
+			&i.Messages,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const startProcessingJob = `-- name: StartProcessingJob :exec
 UPDATE processing_jobs 
 SET started = NOW(),

server/main.go

@@ -20,9 +20,11 @@ import (
 	"log"
 	"log/slog"
 	"os"
+	"sync"
 
 	"git.jmbit.de/jmb/scanfile/server/internal/config"
 	"git.jmbit.de/jmb/scanfile/server/internal/database"
+	"git.jmbit.de/jmb/scanfile/server/internal/processing"
 	"git.jmbit.de/jmb/scanfile/server/internal/server"
 	"git.jmbit.de/jmb/scanfile/server/internal/store"
 	"github.com/spf13/viper"
@@ -32,12 +34,16 @@ func main() {
 	log.SetOutput(os.Stderr)
 	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, nil)))
 	config.ReadConfigFile("")
+  var wg sync.WaitGroup
 	database.Connect()
   database.Ping()
 	store.SetupStore()
+  processing.Setup(&wg)
 	if viper.GetBool("web.tls") {
+    slog.Info("Starting HTTPS server")
 		server.NewServer().ListenAndServeTLS(viper.GetString("web.cert"), viper.GetString("web.key"))
 	} else {
+    slog.Info("Starting HTTP server")
 		server.NewServer().ListenAndServe()
 	}
 }