moving to development in VM
ci/woodpecker/push/woodpecker Pipeline failed Details

templ
Johannes Bülow 2023-08-20 14:46:33 +02:00
parent a4019a8342
commit ac3b2cc828
Signed by untrusted user who does not match committer: jmb
GPG Key ID: B56971CF7B8F83A6
24 changed files with 2150 additions and 231 deletions

1
.gitignore vendored
View File

@ -1,6 +1,7 @@
*.swp
db.sqlite
filestore/*
# config.yaml
### Linux template
*~

View File

@ -1,9 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<dataSource name="db.sqlite">
<database-model serializer="dbm" dbms="SQLITE" family-id="SQLITE" format-version="4.49">
<root id="1">
<ServerVersion>3.40.1</ServerVersion>
</root>
<root id="1"/>
<collation id="2" parent="1" name="RTRIM"/>
<collation id="3" parent="1" name="NOCASE"/>
<collation id="4" parent="1" name="BINARY"/>

View File

@ -9,34 +9,28 @@
</component>
<component name="ChangeListManager">
<list default="true" id="eeedfed2-369e-4948-a46c-18edddaeec9c" name="Changes" comment="">
<change afterPath="$PROJECT_DIR$/.idea/kubernetes-settings.xml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/.woodpecker.yml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/assets/favicon.ico" afterDir="false" />
<change afterPath="$PROJECT_DIR$/assets/logo.svg" afterDir="false" />
<change afterPath="$PROJECT_DIR$/database/fileList.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/database/filesTable.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/templates/filelist.gohtml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/tests/mainTest.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/web/filelist.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/Dockerfile" afterDir="false" />
<change afterPath="$PROJECT_DIR$/compose.yml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/config/config.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/config/defaults.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/config/struct.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/devel-compose.yml" afterDir="false" />
<change afterPath="$PROJECT_DIR$/files/minio.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/podthing/downloadContainer.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/podthing/podmanager.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/web/browser.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" afterPath="$PROJECT_DIR$/.gitignore" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/dataSources/0bebdc19-f5a3-492c-af2a-c7e835e6291d.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/dataSources/0bebdc19-f5a3-492c-af2a-c7e835e6291d.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/dataSources/0bebdc19-f5a3-492c-af2a-c7e835e6291d/storage_v2/_src_/schema/main.uQUzAA.meta" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/dataSources/0bebdc19-f5a3-492c-af2a-c7e835e6291d/storage_v2/_src_/schema/main.uQUzAA.meta" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
<change beforePath="$PROJECT_DIR$/database/fileproperties.go" beforeDir="false" afterPath="$PROJECT_DIR$/database/fileInfo.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/database/sqlitemanager.go" beforeDir="false" afterPath="$PROJECT_DIR$/database/sqlitemanager.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/build.Dockerfile" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/database/fileInfo.go" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/database/fileList.go" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/database/filesTable.go" beforeDir="false" afterPath="$PROJECT_DIR$/database/filesTable.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/files/filemanager.go" beforeDir="false" afterPath="$PROJECT_DIR$/files/filemanager.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/files/staticanalysis.go" beforeDir="false" afterPath="$PROJECT_DIR$/files/staticanalysis.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/go.mod" beforeDir="false" afterPath="$PROJECT_DIR$/go.mod" afterDir="false" />
<change beforePath="$PROJECT_DIR$/go.sum" beforeDir="false" afterPath="$PROJECT_DIR$/go.sum" afterDir="false" />
<change beforePath="$PROJECT_DIR$/main.go" beforeDir="false" afterPath="$PROJECT_DIR$/main.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/sqliteschema.sql" beforeDir="false" afterPath="$PROJECT_DIR$/sqliteschema.sql" afterDir="false" />
<change beforePath="$PROJECT_DIR$/templates/fileview.gohtml" beforeDir="false" afterPath="$PROJECT_DIR$/templates/fileview.gohtml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/templates/footer.gohtml" beforeDir="false" afterPath="$PROJECT_DIR$/templates/footer.gohtml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/templates/header.gohtml" beforeDir="false" afterPath="$PROJECT_DIR$/templates/header.gohtml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/templates/index.gohtml" beforeDir="false" afterPath="$PROJECT_DIR$/templates/index.gohtml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/templates/submitsuccess.gohtml" beforeDir="false" afterPath="$PROJECT_DIR$/templates/submitsuccess.gohtml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/web/fileview.go" beforeDir="false" afterPath="$PROJECT_DIR$/web/fileview.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/web/index.go" beforeDir="false" afterPath="$PROJECT_DIR$/web/index.go" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
@ -68,45 +62,46 @@
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent"><![CDATA[{
"keyToString": {
"DefaultGoTemplateProperty": "Go File",
"DefaultHtmlFileTemplate": "HTML File",
"RunOnceActivity.OpenProjectViewOnStart": "true",
"RunOnceActivity.ShowReadmeOnStart": "true",
"RunOnceActivity.go.formatter.settings.were.checked": "true",
"RunOnceActivity.go.migrated.go.modules.settings": "true",
"RunOnceActivity.go.modules.automatic.dependencies.download": "true",
"SHARE_PROJECT_CONFIGURATION_FILES": "true",
"WebServerToolWindowFactoryState": "false",
"git-widget-placeholder": "main",
"go.import.settings.migrated": "true",
"last_opened_file_path": "/home/johannes/git/go/filegate/templates",
"list.type.of.created.stylesheet": "CSS",
"node.js.detected.package.eslint": "true",
"node.js.detected.package.tslint": "true",
"node.js.selected.package.eslint": "(autodetect)",
"node.js.selected.package.tslint": "(autodetect)",
"nodejs_package_manager_path": "npm",
"ruby.rails.projectView.checked": "true",
"run.code.analysis.last.selected.profile": "pProject Default",
"settings.editor.selected.configurable": "database.query.execution",
"vue.rearranger.settings.migration": "true"
<component name="PropertiesComponent">{
&quot;keyToString&quot;: {
&quot;DefaultGoTemplateProperty&quot;: &quot;Go File&quot;,
&quot;DefaultHtmlFileTemplate&quot;: &quot;HTML File&quot;,
&quot;RunOnceActivity.OpenProjectViewOnStart&quot;: &quot;true&quot;,
&quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
&quot;RunOnceActivity.go.formatter.settings.were.checked&quot;: &quot;true&quot;,
&quot;RunOnceActivity.go.migrated.go.modules.settings&quot;: &quot;true&quot;,
&quot;RunOnceActivity.go.modules.automatic.dependencies.download&quot;: &quot;true&quot;,
&quot;SHARE_PROJECT_CONFIGURATION_FILES&quot;: &quot;true&quot;,
&quot;WebServerToolWindowFactoryState&quot;: &quot;false&quot;,
&quot;git-widget-placeholder&quot;: &quot;main&quot;,
&quot;go.import.settings.migrated&quot;: &quot;true&quot;,
&quot;last_opened_file_path&quot;: &quot;/home/johannes/git/filegate/filegate&quot;,
&quot;list.type.of.created.stylesheet&quot;: &quot;CSS&quot;,
&quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
&quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
&quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
&quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
&quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
&quot;ruby.rails.projectView.checked&quot;: &quot;true&quot;,
&quot;run.code.analysis.last.selected.profile&quot;: &quot;pProject Default&quot;,
&quot;settings.editor.selected.configurable&quot;: &quot;database.query.execution&quot;,
&quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
},
"keyToStringList": {
"DatabaseDriversLRU": [
"sqlite"
&quot;keyToStringList&quot;: {
&quot;DatabaseDriversLRU&quot;: [
&quot;sqlite&quot;
],
"com.intellij.ide.scratch.LRUPopupBuilder$1/": [
"SQLite"
&quot;com.intellij.ide.scratch.LRUPopupBuilder$1/&quot;: [
&quot;SQLite&quot;
],
"com.intellij.ide.scratch.LRUPopupBuilder$1/SQL Dialect": [
"SQLite"
&quot;com.intellij.ide.scratch.LRUPopupBuilder$1/SQL Dialect&quot;: [
&quot;SQLite&quot;
]
}
}]]></component>
}</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$" />
<recent name="$PROJECT_DIR$/templates" />
</key>
<key name="MoveFile.RECENT_KEYS">
@ -164,15 +159,30 @@
</deployment>
<method v="2" />
</configuration>
<configuration default="true" type="docker-deploy" factoryName="docker-compose.yml" temporary="true">
<deployment type="docker-compose.yml">
<settings />
</deployment>
<method v="2" />
</configuration>
<configuration default="true" type="docker-deploy" factoryName="dockerfile" temporary="true">
<deployment type="dockerfile">
<settings />
</deployment>
<method v="2" />
</configuration>
<configuration name="devel-compose.yml: Compose Deployment" type="docker-deploy" factoryName="docker-compose.yml" temporary="true" server-name="Docker">
<deployment type="docker-compose.yml">
<settings>
<option name="sourceFilePath" value="devel-compose.yml" />
</settings>
</deployment>
<method v="2" />
</configuration>
<recent_temporary>
<list>
<item itemvalue="Go Build.go run ." />
<item itemvalue="Docker.devel-compose.yml: Compose Deployment" />
<item itemvalue="Docker.Dockerfile" />
<item itemvalue="Docker.Dockerfile builder" />
<item itemvalue="Docker.Dockerfile builder" />
@ -200,6 +210,17 @@
<workItem from="1691320722229" duration="5760000" />
<workItem from="1691335565690" duration="7160000" />
<workItem from="1691389092008" duration="8058000" />
<workItem from="1691516198036" duration="4781000" />
<workItem from="1691577705602" duration="2040000" />
<workItem from="1691592399619" duration="285000" />
<workItem from="1691593803072" duration="304000" />
<workItem from="1691597758641" duration="2740000" />
<workItem from="1691675493867" duration="3210000" />
<workItem from="1692339071979" duration="4469000" />
<workItem from="1692360995051" duration="1670000" />
<workItem from="1692512812327" duration="2084000" />
<workItem from="1692514947723" duration="1174000" />
<workItem from="1692525884012" duration="3960000" />
</task>
<servers />
</component>

11
Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM git.jmbit.de/filegate/utility-containers:builder-latest AS builder
LABEL authors="Johannes Bülow <johannes.buelow@jmbit.de>"
# Get all the stuff for CGO and Podman bindings
WORKDIR /usr/local/src
COPY ./ ./
RUN go get . && go mod download && go build -o filegate
FROM alpine
LABEL authors="Johannes Bülow <johannes.buelow@jmbit.de>"
COPY --from builder /usr/local/src/filegate /usr/local/bin/
CMD ["/usr/local/bin/filegate"]

96
Vagrantfile vendored Normal file
View File

@ -0,0 +1,96 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
# All Vagrant configuration is done below. The "2" in Vagrant.configure
# configures the configuration version (we support older styles for
# backwards compatibility). Please don't change it unless you know what
# you're doing.
Vagrant.configure("2") do |config|
# The most common configuration options are documented and commented below.
# For a complete reference, please see the online documentation at
# https://docs.vagrantup.com.
# Every Vagrant development environment requires a box. You can search for
# boxes at https://vagrantcloud.com/search.
config.vm.box = "debian/bookworm64"
# Disable automatic box update checking. If you disable this, then
# boxes will only be checked for updates when the user runs
# `vagrant box outdated`. This is not recommended.
# config.vm.box_check_update = false
# Create a forwarded port mapping which allows access to a specific port
# within the machine from a port on the host machine. In the example below,
# accessing "localhost:8080" will access port 80 on the guest machine.
# NOTE: This will enable public access to the opened port
# config.vm.network "forwarded_port", guest: 80, host: 8080
# Create a forwarded port mapping which allows access to a specific port
# within the machine from a port on the host machine and only allow access
# via 127.0.0.1 to disable public access
# config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
# Create a private network, which allows host-only access to the machine
# using a specific IP.
# config.vm.network "private_network", ip: "192.168.33.10"
# Create a public network, which generally matched to bridged network.
# Bridged networks make the machine appear as another physical device on
# your network.
# config.vm.network "public_network"
# Share an additional folder to the guest VM. The first argument is
# the path on the host to the actual folder. The second argument is
# the path on the guest to mount the folder. And the optional third
# argument is a set of non-required options.
# config.vm.synced_folder "../data", "/vagrant_data"
# Disable the default share of the current code directory. Doing this
# provides improved isolation between the vagrant box and your host
# by making sure your Vagrantfile isn't accessable to the vagrant box.
# If you use this you may want to enable additional shared subfolders as
# shown above.
config.vm.synced_folder ".", "/vagrant", disabled: false, type: "rsync"
# Provider-specific configuration so you can fine-tune various
# backing providers for Vagrant. These expose provider-specific options.
# Example for VirtualBox:
#
# config.vm.provider "virtualbox" do |vb|
# # Display the VirtualBox GUI when booting the machine
# vb.gui = true
#
# # Customize the amount of memory on the VM:
# vb.memory = "1024"
# end
#
# View the documentation for the provider you are using for more
# information on available options.
config.vm.provider "libvirt" do |lv|
lv.memory = "4096"
lv.cpus = "4"
end
config.vm.provider "vmware_desktop" do |vd|
vd.memory = "4096"
vd.cpus = "4"
end
# Enable provisioning with a shell script. Additional provisioners such as
# Ansible, Chef, Docker, Puppet and Salt are also available. Please see the
# documentation for more information about their specific syntax and use.
config.vm.provision "shell", inline: <<-SHELL
export DEBIAN_FRONTEND="noninteractive"
apt-get update
apt-get install --no-install-recommends -y software-properties-common
apt-get update
apt-get install --no-install-recommends -y bash binutils ca-certificates clang curl g++ gcc git libc6-dev libssl-dev libdevmapper-dev llvm make pkg-config tzdata uuid libgpgme-dev libbtrfs-dev golang-github-containerd-btrfs-dev postgresql podman
apt-get -y autoremove
apt-get clean -y
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
ln -sf /usr/include/asm-generic /usr/include/asm
wget -O go.tar.gz https://go.dev/dl/go1.21.0.linux-amd64.tar.gz
tar -C /usr/local -xzf go.tar.gz
echo "export PATH=$PATH:/usr/local/go/bin" >> /etc/profile
SHELL
end

View File

@ -1,34 +0,0 @@
FROM docker.io/library/golang
LABEL authors="Johannes Bülow <johannes.buelow@jmbit.de>"
# Get all the stuff for CGO and Podman bindings
RUN export DEBIAN_FRONTEND="noninteractive" \
&& apt-get update \
&& apt-get install --no-install-recommends -y software-properties-common \
&& apt-get update \
&& apt-get install --no-install-recommends -y \
bash \
binutils \
ca-certificates \
clang \
curl \
g++ \
gcc \
git \
libc6-dev \
libssl-dev \
libdevmapper-dev \
llvm \
make \
pkg-config \
tzdata \
uuid \
libgpgme-dev \
libbtrfs-dev \
golang-github-containerd-btrfs-dev \
&& apt-get -y autoremove \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
&& ln -sf /usr/include/asm-generic /usr/include/asm
WORKDIR /usr/local/src
COPY ./ ./
RUN go get . && go mod download && go build -o filegate

24
compose.yml Normal file
View File

@ -0,0 +1,24 @@
---
version: '3'
networks:
browsers:
driver: "bridge"
services:
filegate:
build:
context: .
dockerfile: Dockerfile
image: filegate:devel
ports:
- "8080:8080"
filegate-db:
image: docker.io/postgres:latest
ports:
- "5432"
filegate-storage:
image: quay.io/minio/minio
ports:
- "9000:9000"
- "9001:9001"
command: server /data --console-address ":9001"

46
config/config.go Normal file
View File

@ -0,0 +1,46 @@
package config
import (
"gopkg.in/yaml.v3"
"log"
"os"
)
var Cfg Config
func ParseConfig() {
cfg := getDefaults()
file, err := os.Open("config.yaml")
if err != nil {
log.Println("YIKES: ", err.Error())
}
if err != nil && err.Error() == "open config.yaml: no such file or directory" {
createDefaultConfig()
}
defer func(file *os.File) {
err := file.Close()
if err != nil {
}
}(file)
Cfg = cfg
}
func createDefaultConfig() {
file, err := os.Create("config.yaml")
if err != nil {
log.Fatalf("Could not create default config file: %v", err)
}
defer func(file *os.File) {
err := file.Close()
if err != nil {
}
}(file)
encoder := yaml.NewEncoder(file)
err = encoder.Encode(getDefaults())
if err != nil {
log.Fatalf("failed to encode default Config YAML: %v", err)
}
}

30
config/defaults.go Normal file
View File

@ -0,0 +1,30 @@
package config
import "github.com/google/uuid"
func getDefaults() Config {
cfg := Config{
ServerHostname: "localhost",
ServerPort: 8080,
LogLevel: "info",
DB: struct {
Hostname string
Port int
Owner string
OwnerPassword string
Writer string
WriterPassword string
Reader string
ReaderPassword string
}{Hostname: "localhost", Port: 5432, Owner: "fg-owner", OwnerPassword: uuid.NewString(), Writer: "fg-writer", WriterPassword: uuid.NewString(), Reader: "fg-reader", ReaderPassword: uuid.NewString()},
Minio: struct {
AccessKeyID string
AccessKeySecret string
Hostname string
Port int
Bucket string
UseSSL bool
}{AccessKeyID: "filegate", AccessKeySecret: uuid.NewString(), Hostname: "localhost", Port: 9000, Bucket: "filegate", UseSSL: false},
}
return cfg
}

25
config/struct.go Normal file
View File

@ -0,0 +1,25 @@
package config
type Config struct {
ServerHostname string
ServerPort int
LogLevel string
DB struct {
Hostname string
Port int
Owner string
OwnerPassword string
Writer string
WriterPassword string
Reader string
ReaderPassword string
}
Minio struct {
AccessKeyID string
AccessKeySecret string
Hostname string
Port int
Bucket string
UseSSL bool
}
}

View File

@ -1,57 +0,0 @@
package database
import (
"fmt"
"log"
"time"
)
type Status struct {
id int
Name string
Comment string
}
type FileInfo struct {
Id string
Name string
Url string
Mime string
OriginalName string
Comment string
Size int
Sha256 string
Sha1 string
Md5 string
StatusID int
FileCmd string
Created string
Extension string
Status Status
}
func GetFileInfo(id string) FileInfo {
db := sqliteConnect()
defer sqliteDisconnect(db)
var fileInfo FileInfo
row := db.QueryRow(
`SELECT id, name, url, mime, original_name, comment, size, sha256, sha1, md5, status_id, created
FROM files WHERE id LIKE ?`, id)
log.Println(fmt.Sprintf("Getting info for %s", id))
var createDateUnix int64
if err := row.Scan(&fileInfo.Id, &fileInfo.Name, &fileInfo.Url, &fileInfo.Mime, &fileInfo.OriginalName,
&fileInfo.Comment, &fileInfo.Size, &fileInfo.Sha256, &fileInfo.Sha1, &fileInfo.Md5, &fileInfo.StatusID,
&createDateUnix); err != nil {
log.Println("Error scanning row:", err)
}
createDate := time.Unix(createDateUnix, 0)
fileInfo.Created = createDate.String()
if err := db.QueryRow(`SELECT name, comment FROM status WHERE id = ?`, fileInfo.StatusID).Scan(&fileInfo.Status.Name,
&fileInfo.Status.Comment); err != nil {
log.Println("Error scanning row:", err)
}
return fileInfo
}

View File

@ -1,56 +0,0 @@
package database
import (
"database/sql"
"log"
"time"
)
type ListItem struct {
Id string
Name string
OriginalName string
Size int64
MimeType string
Status string
Comment string
Created string
}
func GetFileList(page int, count int) ([]ListItem, error) {
db := sqliteConnect()
var returnList []ListItem
firstEntry := (page - 1) * count
lastEntry := page * count
query, err := db.Query(`SELECT files.id, files.name, files.original_name, files.size, files.mime, status.name, files.comment, files.created
FROM files, status
WHERE files.status_id = status.id
OFFSET ? ROWS FETCH NEXT ? ROWS ONLY;`, firstEntry, lastEntry)
if err != nil {
log.Println(err)
return nil, err
}
defer func(query *sql.Rows) {
err := query.Close()
if err != nil {
log.Println(err)
}
}(query)
defer sqliteDisconnect(db)
for query.Next() {
var entry ListItem
var unixTime int64
err := query.Scan(&entry.Id, &entry.Name, &entry.OriginalName, &entry.Size, &entry.MimeType, &entry.Status, &entry.Comment, &unixTime)
if err != nil {
log.Println(err)
return nil, err
}
entry.Created = time.Unix(unixTime, 0).String()
}
if err := query.Err(); err != nil {
log.Println(err)
return nil, err
}
return returnList, nil
}

View File

@ -1,10 +1,47 @@
package database
import (
"database/sql"
"fmt"
"log"
"time"
)
type ListItem struct {
Id string
Name string
OriginalName string
Size int64
MimeType string
Status string
Comment string
Created string
}
type Status struct {
id int
Name string
Comment string
}
type FileInfo struct {
Id string
Name string
Url string
Mime string
OriginalName string
Comment string
Size int
Sha256 string
Sha1 string
Md5 string
StatusID int
FileCmd string
Created string
Extension string
Status Status
}
func UpdateOriginalName(id string, originalName string) {
db := sqliteConnect()
_, err := db.Exec(`UPDATE files SET original_name = ? WHERE id LIKE ?;`, originalName, id)
@ -41,3 +78,67 @@ func CountFilesEntries() int {
}
return count
}
func GetFileInfo(id string) FileInfo {
db := sqliteConnect()
defer sqliteDisconnect(db)
var fileInfo FileInfo
row := db.QueryRow(
`SELECT id, name, url, mime, original_name, comment, size, sha256, sha1, md5, status_id, created
FROM files WHERE id LIKE ?`, id)
log.Println(fmt.Sprintf("Getting info for %s", id))
var createDateUnix int64
if err := row.Scan(&fileInfo.Id, &fileInfo.Name, &fileInfo.Url, &fileInfo.Mime, &fileInfo.OriginalName,
&fileInfo.Comment, &fileInfo.Size, &fileInfo.Sha256, &fileInfo.Sha1, &fileInfo.Md5, &fileInfo.StatusID,
&createDateUnix); err != nil {
log.Println("Error scanning row:", err)
}
createDate := time.Unix(createDateUnix, 0)
fileInfo.Created = createDate.String()
if err := db.QueryRow(`SELECT name, comment FROM status WHERE id = ?`, fileInfo.StatusID).Scan(&fileInfo.Status.Name,
&fileInfo.Status.Comment); err != nil {
log.Println("Error scanning row:", err)
}
return fileInfo
}
func GetFileList(page int, count int) ([]ListItem, error) {
db := sqliteConnect()
var returnList []ListItem
firstEntry := (page - 1) * count
lastEntry := page * count
query, err := db.Query(`SELECT files.id, files.name, files.original_name, files.size, files.mime, status.name, files.comment, files.created
FROM files, status
WHERE files.status_id = status.id
OFFSET ? ROWS FETCH NEXT ? ROWS ONLY;`, firstEntry, lastEntry)
if err != nil {
log.Println(err)
return nil, err
}
defer func(query *sql.Rows) {
err := query.Close()
if err != nil {
log.Println(err)
}
}(query)
defer sqliteDisconnect(db)
for query.Next() {
var entry ListItem
var unixTime int64
err := query.Scan(&entry.Id, &entry.Name, &entry.OriginalName, &entry.Size, &entry.MimeType, &entry.Status, &entry.Comment, &unixTime)
if err != nil {
log.Println(err)
return nil, err
}
entry.Created = time.Unix(unixTime, 0).String()
}
if err := query.Err(); err != nil {
log.Println(err)
return nil, err
}
return returnList, nil
}

17
devel-compose.yml Normal file
View File

@ -0,0 +1,17 @@
---
version: '3'
networks:
browsers:
driver: "bridge"
services:
filegate-db:
image: docker.io/postgres:latest
ports:
- "5432"
filegate-storage:
image: quay.io/minio/minio
ports:
- "9000:9000"
- "9001:9001"
command: server /data --console-address ":9001"

View File

@ -62,6 +62,10 @@ func DownloadFile(rawURL string, id string) error {
}
}(targetFile)
contentType := response.Header.Get("content-type")
if contentType == "" {
contentType = "application/octet-stream"
}
_, err = io.Copy(targetFile, response.Body)
if err != nil {
log.Println(err)

36
files/minio.go Normal file
View File

@ -0,0 +1,36 @@
package files
import (
"context"
"fmt"
"git.jmbit.de/filegate/filegate/config"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"log"
)
var minioConfig = config.Cfg.Minio
func minioConnect() *minio.Client {
endpoint := fmt.Sprintf("%s:%d", minioConfig.Hostname, minioConfig.Port)
accessKeyID := minioConfig.AccessKeyID
accessKeySecret := minioConfig.AccessKeySecret
minioClient, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, accessKeySecret, ""),
Secure: config.Cfg.Minio.UseSSL,
})
if err != nil {
log.Fatalf("Could not connect to Minio storage: %v", err)
}
return minioClient
}
func minioSetup() {
ctx := context.Background()
client := minioConnect()
bucketName := minioConfig.Bucket
err = client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: ""})
}

132
go.mod
View File

@ -3,35 +3,151 @@ module git.jmbit.de/filegate/filegate
go 1.20
require (
github.com/containers/podman/v4 v4.6.0
github.com/gin-gonic/gin v1.9.1
github.com/google/uuid v1.3.0
github.com/mattn/go-sqlite3 v1.14.16
github.com/mattn/go-sqlite3 v1.14.17
github.com/minio/minio-go/v7 v7.0.62
gopkg.in/yaml.v3 v3.0.1
)
require (
dario.cat/mergo v1.0.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/Microsoft/hcsshim v0.10.0-rc.8 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/bytedance/sonic v1.9.1 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
github.com/chzyer/readline v1.5.1 // indirect
github.com/cilium/ebpf v0.9.1 // indirect
github.com/container-orchestrated-devices/container-device-interface v0.5.4 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/containerd/containerd v1.7.2 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/containers/buildah v1.31.0 // indirect
github.com/containers/common v0.55.2 // indirect
github.com/containers/image/v5 v5.26.1 // indirect
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
github.com/containers/ocicrypt v1.1.7 // indirect
github.com/containers/psgo v1.8.0 // indirect
github.com/containers/storage v1.48.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/cyberphone/json-canonicalization v0.0.0-20230514072755-504adb8a8af1 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/disiqueira/gotree/v3 v3.0.2 // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v24.0.2+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-openapi/analysis v0.21.4 // indirect
github.com/go-openapi/errors v0.20.3 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/loads v0.21.2 // indirect
github.com/go-openapi/runtime v0.26.0 // indirect
github.com/go-openapi/spec v0.20.9 // indirect
github.com/go-openapi/strfmt v0.21.7 // indirect
github.com/go-openapi/swag v0.22.4 // indirect
github.com/go-openapi/validate v0.22.1 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.14.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-containerregistry v0.15.2 // indirect
github.com/google/go-intervals v0.0.2 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/schema v1.2.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/jinzhu/copier v0.3.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/letsencrypt/boulder v0.0.0-20230213213521-fdfea0d469b6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/manifoldco/promptui v0.9.0 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-shellwords v1.0.12 // indirect
github.com/miekg/pkcs11 v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mistifyio/go-zfs/v3 v3.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/sys/mountinfo v0.6.2 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
github.com/opencontainers/runc v1.1.7 // indirect
github.com/opencontainers/runtime-spec v1.1.0-rc.3 // indirect
github.com/opencontainers/runtime-tools v0.9.1-0.20230317050512-e931285f4b69 // indirect
github.com/opencontainers/selinux v1.11.0 // indirect
github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/sftp v1.13.5 // indirect
github.com/proglottis/gpgme v0.1.3 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/sigstore/fulcio v1.3.1 // indirect
github.com/sigstore/rekor v1.2.2-0.20230601122533-4c81ff246d12 // indirect
github.com/sigstore/sigstore v1.7.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect
github.com/sylabs/sif/v2 v2.11.5 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/tchap/go-patricia/v2 v2.3.1 // indirect
github.com/theupdateframework/go-tuf v0.5.2 // indirect
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
github.com/vbauerster/mpb/v8 v8.4.0 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
go.mongodb.org/mongo-driver v1.11.3 // indirect
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.9.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
golang.org/x/crypto v0.12.0 // indirect
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/term v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/tools v0.9.3 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
google.golang.org/grpc v1.55.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gopkg.in/go-jose/go-jose.v2 v2.6.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

1414
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -3,6 +3,7 @@ package main
import (
"embed"
"fmt"
"git.jmbit.de/filegate/filegate/config"
"git.jmbit.de/filegate/filegate/database"
"git.jmbit.de/filegate/filegate/files"
"git.jmbit.de/filegate/filegate/web"
@ -23,7 +24,10 @@ func main() {
if runtime.GOOS == "windows" {
fmt.Println("This Software is not supported on Windows")
}
config.ParseConfig()
addr := fmt.Sprintf("%s:%d", config.Cfg.ServerHostname, config.Cfg.ServerPort)
database.Setup()
// podthing.TestPodman()
files.StorageDirectory()
router := gin.Default()
templates := template.Must(template.New("").ParseFS(templatesFS, "templates/*.gohtml"))
@ -38,6 +42,8 @@ func main() {
router.GET("/file/:id", web.GetFileView)
router.GET("/file/", web.GetFileList)
router.POST("/file/new", web.PostSubmitFileUrl)
router.POST("/file/upload", web.PostUploadFile)
router.GET("/browser/:id/")
router.GET("favicon.ico", func(c *gin.Context) {
file, _ := assetsFS.ReadFile("assets/favicon.ico")
c.Data(
@ -46,7 +52,7 @@ func main() {
file,
)
})
err = router.Run("127.0.0.1:8080")
err = router.Run(addr)
if err != nil {
log.Fatal(err)
}

57
pgsetup.sql Normal file
View File

@ -0,0 +1,57 @@
-- DROP all tables (Only for development purposes
DROP TABLE IF EXISTS status;
DROP TABLE IF EXISTS file;
DROP TABLE IF EXISTS file_properties;
DROP TABLE IF EXISTS dynamic_analysis;
DROP TABLE IF EXISTS file_attachment;
CREATE TABLE IF NOT EXISTS status
(
id INTEGER PRIMARY KEY,
name VARCHAR[64],
comment VARCHAR[2048]
);
CREATE TABLE IF NOT EXISTS file
(
id VARCHAR[36] PRIMARY KEY,
name VARCHAR[64],
mime VARCHAR,
comment VARCHAR[2048],
status INT,
created BIGINT,
properties INT,
FOREIGN KEY (status)
REFERENCES status (id),
FOREIGN KEY (properties)
REFERENCES file_properties (id)
);
CREATE TABLE IF NOT EXISTS file_properties
(
id SERIAL PRIMARY KEY,
sha256 VARCHAR[64],
sha1 VARCHAR[40],
md5 VARCHAR[32],
extension VARCHAR[80],
file_cmd VARCHAR[512],
original_name VARCHAR[256],
url VARCHAR[2048],
size BIGINT
);
CREATE TABLE IF NOT EXISTS file_attachment
(
id SERIAL PRIMARY KEY,
type VARCHAR[32],
comment VARCHAR[2048],
file_id VARCHAR[36],
FOREIGN KEY (file_id)
REFERENCES file (id)
);
INSERT INTO status (id, name, comment)
VALUES (0, 'New', 'Freshly uploaded file');
INSERT INTO status (id, name, comment)
VALUES (1, 'Processing', 'File is now being analyzed');
INSERT INTO status (id, name, comment)
VALUES (2, 'Denied', 'File download was denied');
INSERT INTO status (id, name, comment)
VALUES (3, 'Granted', 'File download was granted');

View File

@ -0,0 +1,41 @@
package podthing
import (
"github.com/containers/podman/v4/pkg/bindings/containers"
"github.com/containers/podman/v4/pkg/bindings/images"
"github.com/containers/podman/v4/pkg/specgen"
"log"
)
func CreateDownloadContainer(user string, passwd string, folder string) string {
image := "docker.io/linuxserver/firefox:version-116.0-r0"
conn := socketConnection()
_, err := images.Pull(conn, image, nil)
if err != nil {
log.Println(err)
}
s := specgen.NewSpecGenerator(image, false)
s.Env["CUSTOM_USER"] = user
s.Env["PASSWORD"] = passwd
s.Env["START_DOCKER"] = "False"
s.Env["TITLE"] = "Filegate Download Browser"
s.Env["SUBFOLDER"] = folder
createResponse, err := containers.CreateWithSpec(conn, s, nil)
if err != nil {
log.Println(err)
}
if err := containers.Start(conn, createResponse.ID, nil); err != nil {
log.Println(err)
}
return createResponse.ID
}
func DestroyDownloadContainer(id string) {
conn := socketConnection()
if err := containers.Kill(conn, id, nil); err != nil {
log.Println(err)
}
if err, _ := containers.Remove(conn, id, nil); err != nil {
log.Println(err)
}
}

45
podthing/podmanager.go Normal file
View File

@ -0,0 +1,45 @@
package podthing
import (
"context"
"fmt"
"github.com/containers/podman/v4/pkg/bindings"
"github.com/containers/podman/v4/pkg/bindings/containers"
"github.com/containers/podman/v4/pkg/bindings/images"
"github.com/containers/podman/v4/pkg/specgen"
"log"
"os"
)
func socketConnection() context.Context {
uri := fmt.Sprintf("unix:///run/user/%d/podman/podman.sock", os.Getuid())
conn, err := bindings.NewConnection(context.Background(), uri)
if err != nil {
log.Println(err)
}
return conn
}
func TestPodman() {
conn := socketConnection()
_, err := images.Pull(conn, "quay.io/libpod/alpine_nginx", nil)
if err != nil {
log.Fatal(err)
}
s := specgen.NewSpecGenerator("quay.io/libpod/alpine_nginx", false)
s.Name = "fg-test"
createResponse, err := containers.CreateWithSpec(conn, s, nil)
if err != nil {
log.Fatal(err)
}
if err := containers.Start(conn, createResponse.ID, nil); err != nil {
log.Fatal(err)
}
if err := containers.Kill(conn, createResponse.ID, nil); err != nil {
log.Fatal(err)
}
if err, _ := containers.Remove(conn, createResponse.ID, nil); err != nil {
log.Fatal(err)
}
}

View File

@ -26,8 +26,6 @@
<form action="/browser/new" method="POST">
<label for="name">Enter Name of the File</label>
<input type="text" name="name" id="name" class="form-control" required>
<label for="url">Enter an URL to start from</label>
<input type="text" name="url" id="url" class="form-control" required>
<label for="comment">Add a Comment with Context</label>
<input type="text" name="comment" id="comment" class="form-control" required>
<input type="submit" value="Start Browser" class="btn btn-primary">

13
web/browser.go Normal file
View File

@ -0,0 +1,13 @@
package web
import (
"git.jmbit.de/filegate/filegate/podthing"
"github.com/google/uuid"
)
func NewBrowser() (string, string, string) {
passwd := uuid.NewString()
folderId := uuid.NewString()
browserPod := podthing.CreateDownloadContainer("user", passwd, folderId)
return browserPod, folderId, passwd
}