diff --git a/config.toml b/config.toml index 5481256..4edc915 100644 --- a/config.toml +++ b/config.toml @@ -17,4 +17,4 @@ debug = true path = "./storage/files/" [processing] -executors = 5 +maxparallel = 5 diff --git a/go.mod b/go.mod index d4e1c79..e08c271 100644 --- a/go.mod +++ b/go.mod @@ -14,21 +14,14 @@ require ( github.com/lib/pq v1.10.9 github.com/spf13/viper v1.20.1 golang.org/x/crypto v0.38.0 - xorm.io/xorm v1.3.9 ) require ( github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect - github.com/goccy/go-json v0.10.5 // indirect - github.com/golang/snappy v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/mattn/go-sqlite3 v1.14.27 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/sagikazarmark/locafero v0.9.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -36,10 +29,9 @@ require ( github.com/spf13/cast v1.8.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/syndtr/goleveldb v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/sync v0.14.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.25.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - xorm.io/builder v0.3.13 // indirect ) diff --git a/go.sum b/go.sum index 26f70a4..dad13ab 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,3 @@ -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= -gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= github.com/Oudwins/tailwind-merge-go v0.2.1 h1:jxRaEqGtwwwF48UuFIQ8g8XT7YSualNuGzCvQ89nPFE= github.com/Oudwins/tailwind-merge-go v0.2.1/go.mod h1:kkZodgOPvZQ8f7SIrlWkG/w1g9JTbtnptnePIh3V72U= github.com/a-h/templ v0.3.865 h1:nYn5EWm9EiXaDgWcMQaKiKvrydqgxDUtT1+4zU2C43A= @@ -9,26 +5,14 @@ github.com/a-h/templ v0.3.865/go.mod h1:oLBbZVQ6//Q6zpvSMPTuBK0F3qOtBdFBcGRspcT+ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= -github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -39,8 +23,6 @@ github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzq github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -51,36 +33,16 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.27 h1:drZCnuvf37yPfs95E5jd9s3XhdVWLal+6BOK6qrv6IU= -github.com/mattn/go-sqlite3 v1.14.27/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k= @@ -102,61 +64,19 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= -golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= -golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= -lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= -modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= -modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= -modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= -modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0= -modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= -modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk= -modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.20.4 h1:J8+m2trkN+KKoE7jglyHYYYiaq5xmz2HoHJIiBlRzbE= -modernc.org/sqlite v1.20.4/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A= -modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= -modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= -modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= -modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -xorm.io/builder v0.3.13 h1:a3jmiVVL19psGeXx8GIurTp7p0IIgqeDmwhcR6BAOAo= -xorm.io/builder v0.3.13/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= -xorm.io/xorm v1.3.9 h1:TUovzS0ko+IQ1XnNLfs5dqK1cJl1H5uHpWbWqAQ04nU= -xorm.io/xorm v1.3.9/go.mod h1:LsCCffeeYp63ssk0pKumP6l96WZcHix7ChpurcLNuMw= diff --git a/server/internal/database/createFile.go b/server/internal/database/createFile.go index 65a7f90..ef2db99 100644 --- a/server/internal/database/createFile.go +++ b/server/internal/database/createFile.go @@ -1,27 +1,33 @@ package database - import ( + "context" "encoding/hex" "log/slog" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "golang.org/x/crypto/blake2b" + "git.jmbit.de/jmb/scanfile/server/internal/sqlc" "git.jmbit.de/jmb/scanfile/server/internal/store" ) - // CreateFile() creates the filesystem object and the DB entry for a file -func CreateFile(name string, fileBytes []byte, size int64) (File, error) { - file := File{ +func CreateFile(ctx context.Context, name string, fileBytes []byte, size int64) (File, error) { + file := sqlc.File{ Name: name, Size: size, } bl2hash := blake2b.Sum256(fileBytes) - file.Blake2 = hex.EncodeToString(bl2hash[:]) - fileExists, err := FileAlreadyExists(file.Blake2) + file.Blake2 = bl2hash[:] + queries := sqlc.New(pool) + fileExists, err := queries.GetFileByBlake2() + if err == pgx.ErrNoRows { + slog.Info("File already exists", "file-uuid", ) + } + if err != nil { slog.Error("Could not check if a duplicate exists", "error", err) err = nil diff --git a/server/internal/database/db.go b/server/internal/database/db.go index 55fbdf0..bfffa6b 100644 --- a/server/internal/database/db.go +++ b/server/internal/database/db.go @@ -1,22 +1,22 @@ package database import ( + "context" "fmt" "log/slog" "net/url" + "github.com/jackc/pgx/v5/pgxpool" _ "github.com/lib/pq" "github.com/spf13/viper" - "xorm.io/xorm" - "xorm.io/xorm/log" - "xorm.io/xorm/names" ) -var engine *xorm.Engine -func Connect() (*xorm.Engine, error) { - if engine != nil { - return engine, nil +var pool *pgxpool.Pool + +func Connect() (*pgxpool.Pool, error) { + if pool != nil { + return pool, nil } dburl := url.URL{ Scheme: "postgres", @@ -24,27 +24,19 @@ func Connect() (*xorm.Engine, error) { Host: fmt.Sprintf("%s:%d", viper.GetString("db.host"), viper.GetInt("db.port")), Path: fmt.Sprintf("/%s", viper.GetString("db.database")), } - engi, err := xorm.NewEngine("postgres", dburl.String()) + conn, err := pgxpool.New(context.Background(), dburl.String()) if err != nil { slog.Error("Could not connect to DB", "url", dburl.Redacted(), "error", err) return nil, err } - engi.SetMapper(names.LintGonicMapper) - err = engi.Sync(new(User), new(File)) - if err != nil { - slog.Error("Error syncing DB schema", "error", err) - } - if viper.GetBool("db.debug") { - engi.SetLogLevel(log.LOG_DEBUG) - engi.ShowSQL(true) - } - engine = engi - return engi, nil + pool = conn + + return conn, nil } func Ping() error { - err := engine.DB().Ping() + err := pool.Ping(context.Background()) if err != nil { slog.Error("Error connecting to DB", "error", err) return err diff --git a/server/internal/database/queries-processing_jobs.sql b/server/internal/database/queries-processing_jobs.sql index f38867d..1c3a051 100644 --- a/server/internal/database/queries-processing_jobs.sql +++ b/server/internal/database/queries-processing_jobs.sql @@ -15,3 +15,12 @@ UPDATE processing_jobs SET completed = NOW(), status = "completed" WHERE id = $1; + +-- name: GetJobsForFile :many +SELECT * FROM processing_jobs +WHERE file_id = $1; + +-- name: GetJob :one +SELECT * FROM processing_jobs +WHERE id = $1 +LIMIT 1; diff --git a/server/internal/database/schema.sql b/server/internal/database/schema.sql index bfc9ce4..7a25a94 100644 --- a/server/internal/database/schema.sql +++ b/server/internal/database/schema.sql @@ -1,21 +1,66 @@ +-- Extensions CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +-- Tables CREATE TABLE IF NOT EXISTS files ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), name TEXT NOT NULL, description TEXT, mimetype TEXT NOT NULL, - size INTEGER NOT NULL, + size BIGINT NOT NULL, blake2 BYTEA NOT NULL UNIQUE, created TIMESTAMP DEFAULT NOW() NOT NULL, updated TIMESTAMP DEFAULT NOW() NOT NULL ); CREATE TABLE IF NOT EXISTS processing_jobs ( - id SERIAL PRIMARY KEY, - file_id UUID REFERENCES files (id), + id BIGSERIAL PRIMARY KEY, + file_id UUID REFERENCES files (id) ON DELETE CASCADE, created TIMESTAMP DEFAULT NOW(), started TIMESTAMP, completed TIMESTAMP, status TEXT, - job_type TEXT + job_type TEXT, + error TEXT, + messages JSONB ); + +CREATE TABLE IF NOT EXISTS diec ( + id BIGSERIAL PRIMARY KEY, + file_id UUID REFERENCES files (id) ON DELETE CASCADE, + data JSONB, + created TIMESTAMP DEFAULT NOW() NOT NULL, + updated TIMESTAMP DEFAULT NOW() NOT NULL +); + +CREATE TABLE IF NOT EXISTS msoffice_oleid ( + id BIGSERIAL PRIMARY KEY, + file_id UUID REFERENCES files (id) ON DELETE CASCADE, + data JSONB, + created TIMESTAMP DEFAULT NOW() NOT NULL, + updated TIMESTAMP DEFAULT NOW() NOT NULL +); + +CREATE TABLE IF NOT EXISTS msoffice_olevba ( + id BIGSERIAL PRIMARY KEY, + file_id UUID REFERENCES files (id) ON DELETE CASCADE, + data JSONB, + created TIMESTAMP DEFAULT NOW() NOT NULL, + updated TIMESTAMP DEFAULT NOW() NOT NULL +); + +CREATE TABLE IF NOT EXISTS msoffice_mraptor ( + id BIGSERIAL PRIMARY KEY, + file_id UUID REFERENCES files (id) ON DELETE CASCADE, + data JSONB, + created TIMESTAMP DEFAULT NOW() NOT NULL, + updated TIMESTAMP DEFAULT NOW() NOT NULL +); + +-- Indices +-- Since tables will be heavily accessed by file_id, there should be indices for them +CREATE INDEX idx_diec_file_id ON diec (file_id); +CREATE INDEX idx_processing_jobs_file_id ON processing_jobs (file_id); +CREATE INDEX idx_msoffice_oleid_file_id ON msoffice_oleid (file_id); +CREATE INDEX idx_msoffice_olevba_file_id ON msoffice_olevba (file_id); +CREATE INDEX idx_msoffice_mraptor_file_id ON msoffice_mraptor (file_id); diff --git a/server/internal/processing/processing.go b/server/internal/processing/processing.go index ea1556c..7de2b05 100644 --- a/server/internal/processing/processing.go +++ b/server/internal/processing/processing.go @@ -1,4 +1,21 @@ package processing +import ( + "sync" + "time" + + "github.com/spf13/viper" +) + +var semaphore chan struct{} +var swg *sync.WaitGroup +//Used to determine if a task was started by a previous instance that stalled out or died +var startup time.Time + +func Setup(wg *sync.WaitGroup) { + semaphore = make(chan struct{}, viper.GetInt("processing.maxparallel")) + startup = time.Now() +} + func Submit() diff --git a/server/internal/processing/types.go b/server/internal/processing/types.go new file mode 100644 index 0000000..ece7219 --- /dev/null +++ b/server/internal/processing/types.go @@ -0,0 +1,14 @@ +package processing + +// Microsoft Office Document +const TypeMSOffice = "MSOffice" +// Microsoft Windows Portable Executable +const TypePE = "PE" +// Linux/UNIX Executable Linkable Format +const TypeELF = "ELF" +// Java Archive (JAR) +const TypeJAR = "JAR" +// Archives (compressed etc.) +const TypeArchive = "Archive" +// Anything not implemented (yet) +const TypeOther = "Other" diff --git a/server/internal/sqlc/models.go b/server/internal/sqlc/models.go index 5188ca1..f8420d5 100644 --- a/server/internal/sqlc/models.go +++ b/server/internal/sqlc/models.go @@ -13,7 +13,7 @@ type File struct { Name string Description pgtype.Text Mimetype string - Size int32 + Size int64 Blake2 []byte Created pgtype.Timestamp Updated pgtype.Timestamp @@ -27,4 +27,6 @@ type ProcessingJob struct { Completed pgtype.Timestamp Status pgtype.Text JobType pgtype.Text + Error pgtype.Text + Messages []byte } diff --git a/server/internal/sqlc/queries-files.sql.go b/server/internal/sqlc/queries-files.sql.go index 581253e..8a04c28 100644 --- a/server/internal/sqlc/queries-files.sql.go +++ b/server/internal/sqlc/queries-files.sql.go @@ -25,7 +25,7 @@ type CreateFileParams struct { Name string Description pgtype.Text Mimetype string - Size int32 + Size int64 Blake2 []byte } diff --git a/server/internal/sqlc/queries-processing_jobs.sql.go b/server/internal/sqlc/queries-processing_jobs.sql.go index e8efe59..6c4a79b 100644 --- a/server/internal/sqlc/queries-processing_jobs.sql.go +++ b/server/internal/sqlc/queries-processing_jobs.sql.go @@ -15,7 +15,7 @@ const createProcessingJob = `-- name: CreateProcessingJob :one INSERT INTO processing_jobs ( file_id, job_type ) VALUES ($1,$2 ) -RETURNING id, file_id, created, started, completed, status, job_type +RETURNING id, file_id, created, started, completed, status, job_type, error, messages ` type CreateProcessingJobParams struct { @@ -34,6 +34,8 @@ func (q *Queries) CreateProcessingJob(ctx context.Context, arg CreateProcessingJ &i.Completed, &i.Status, &i.JobType, + &i.Error, + &i.Messages, ) return i, err } @@ -50,6 +52,64 @@ func (q *Queries) FinishProcessingJob(ctx context.Context, id int32) error { return err } +const getJob = `-- name: GetJob :one +SELECT id, file_id, created, started, completed, status, job_type, error, messages FROM processing_jobs +WHERE id = $1 +LIMIT 1 +` + +func (q *Queries) GetJob(ctx context.Context, id int32) (ProcessingJob, error) { + row := q.db.QueryRow(ctx, getJob, id) + var i ProcessingJob + err := row.Scan( + &i.ID, + &i.FileID, + &i.Created, + &i.Started, + &i.Completed, + &i.Status, + &i.JobType, + &i.Error, + &i.Messages, + ) + return i, err +} + +const getJobsForFile = `-- name: GetJobsForFile :many +SELECT id, file_id, created, started, completed, status, job_type, error, messages FROM processing_jobs +WHERE file_id = $1 +` + +func (q *Queries) GetJobsForFile(ctx context.Context, fileID pgtype.UUID) ([]ProcessingJob, error) { + rows, err := q.db.Query(ctx, getJobsForFile, fileID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ProcessingJob + for rows.Next() { + var i ProcessingJob + if err := rows.Scan( + &i.ID, + &i.FileID, + &i.Created, + &i.Started, + &i.Completed, + &i.Status, + &i.JobType, + &i.Error, + &i.Messages, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const startProcessingJob = `-- name: StartProcessingJob :exec UPDATE processing_jobs SET started = NOW(), diff --git a/server/main.go b/server/main.go index 0040716..93df787 100644 --- a/server/main.go +++ b/server/main.go @@ -20,9 +20,11 @@ import ( "log" "log/slog" "os" + "sync" "git.jmbit.de/jmb/scanfile/server/internal/config" "git.jmbit.de/jmb/scanfile/server/internal/database" + "git.jmbit.de/jmb/scanfile/server/internal/processing" "git.jmbit.de/jmb/scanfile/server/internal/server" "git.jmbit.de/jmb/scanfile/server/internal/store" "github.com/spf13/viper" @@ -32,12 +34,16 @@ func main() { log.SetOutput(os.Stderr) slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, nil))) config.ReadConfigFile("") + var wg sync.WaitGroup database.Connect() database.Ping() store.SetupStore() + processing.Setup(&wg) if viper.GetBool("web.tls") { + slog.Info("Starting HTTPS server") server.NewServer().ListenAndServeTLS(viper.GetString("web.cert"), viper.GetString("web.key")) } else { + slog.Info("Starting HTTP server") server.NewServer().ListenAndServe() } }