adding back filesystem backend as an option

.gitignore

@@ -2,8 +2,7 @@
 db.sqlite
 filestore/*
 filegate
-config.yaml
-oldconfig.yaml
+.vagrant/*
 
 ### Linux template
 *~

Makefile

@@ -19,3 +19,10 @@ templ:
 deps:
 	go mod download
 	go mod tidy
+
+install:
+	sudo useradd -d /var/lib/filegate -c"Filegate Service User" -rmUF filegate
+	sudo mkdir -p /etc/filegate
+	sudo install -o root -g filegate -m 750 ./filegate /usr/local/filegate
+	./filegate genconfig
+	sudo cp config.yaml /etc/filegate

Vagrantfile

@@ -0,0 +1,127 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "debian/bookworm64"
+  config.vm.hostname = "filegate"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # NOTE: This will enable public access to the opened port
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine and only allow access
+  # via 127.0.0.1 to disable public access
+  # config.vm.network "forwarded_port", guest: 8080, host: 8080, host_ip: "127.0.0.1"
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+
+  # Disable the default share of the current code directory. Doing this
+  # provides improved isolation between the vagrant box and your host
+  # by making sure your Vagrantfile isn't accessable to the vagrant box.
+  # If you use this you may want to enable additional shared subfolders as
+  # shown above.
+  # config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder ".", "/usr/local/src/filegate", type: "rsync", rsync__auto: true
+
+
+  # Configure the window for gatling to coalesce writes.
+  if Vagrant.has_plugin?("vagrant-gatling-rsync")
+    config.gatling.latency = 2.5
+    config.gatling.time_format = "%H:%M:%S"
+  end
+
+  # Automatically sync when machines with rsync folders come up.
+  config.gatling.rsync_on_startup = true
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb|
+    # Display the VirtualBox GUI when booting the machine
+    vb.gui = true
+  
+    # Customize the amount of memory on the VM:
+    vb.memory = "4096"
+    vb.cpus = 4
+  end
+
+  config.vm.provider "libvirt" do |lv|
+    lv.memory = "4096"
+    lv.cpus = 4
+  end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Ansible, Chef, Docker, Puppet and Salt are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  config.vm.provision "shell", inline: <<-SHELL
+    apt-get update
+    apt-get upgrade -y
+    apt-get install vim curl git wget qemu-guest-agent nfs-common podman pwgen postgresql -y
+
+    ## GOLANG
+    wget https://go.dev/dl/go1.21.5.linux-amd64.tar.gz
+    rm -rf /usr/local/go && tar -C /usr/local -xzf go1.21.5.linux-amd64.tar.gz
+    rm go1.21.5.linux-amd64.tar.gz
+    echo 'export PATH=$PATH:/usr/local/go/bin' >> /etc/profile
+    export PATH=$PATH:/usr/local/go/bin
+
+    ## MINIO
+    wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio_20231223071911.0.0_amd64.deb -O minio.deb
+    dpkg -i minio.deb
+    rm minio.deb
+    cp /usr/local/src/filegate/vagrant-minio-defaults /etc/default/minio
+    systemctl enable --now minio
+    journalctl -xu minio.service
+    mc alias set admin http://localhost:9000 minioadmin Brethren2-Nativity-Mashing-Catalog-Freeway 
+    mc admin info admin
+    mc admin user svcacct add \
+      --access-key "lZvkgrfXNbEMye6BSf6s" \
+      --secret-key "U109MtkE1jcc6qm3SIGk3IEZsq1cl8vTxqIRr3ZH"\
+      admin filegate
+    mc admin user ls admin
+
+    ## Filegate
+    cp /usr/local/src/filegate/filegate.service /etc/systemd/system/
+    systemctl daemon-reload
+    cd /usr/local/src/filegate/ && make release &&
+    sudo useradd -d /var/lib/filegate -c"Filegate Service User" -rmUF filegate
+	  sudo mkdir -p /etc/filegate
+	  sudo install -o root -g filegate -m 750 /usr/local/src/filegate/filegate /usr/local/filegate
+	  sudo cp /usr/local/src/filegate/config.yaml /etc/filegate/
+  SHELL
+end

cmd/root.go

@@ -25,11 +25,13 @@ import (
 	"github.com/spf13/viper"
 
 	"git.jmbit.de/filegate/filegate/db"
+	"git.jmbit.de/filegate/filegate/files"
 	"git.jmbit.de/filegate/filegate/utils"
 	"git.jmbit.de/filegate/filegate/web"
 )
 
 var cfgFile string
+var localfs string
 var production bool
 
 // rootCmd represents the base command when called without any subcommands
@@ -46,6 +48,15 @@ Frontend tooling and Bulma CSS`,
 	// has an action associated with it:
 	Run: func(cmd *cobra.Command, args []string) {
 		db.ConnectDB()
+		// Check if local Filesystem should be used
+		if localfs == "" || viper.GetString("localfs") == "" {
+			files.MinioConnect()
+			files.MinioSetup()
+		} else {
+			// override any Filesystem declared via config file with the cli arg
+			viper.Set("localfs", localfs)
+
+		}
 		web.Router(production)
 	},
 }
@@ -67,9 +78,11 @@ func init() {
 	// will be global for your application.
 
 	rootCmd.PersistentFlags().
-		StringVar(&cfgFile, "config", "", "config file (default is /etc/filegate.yaml)")
+		StringVarP(&cfgFile, "config", "c", "", "config file (default is /etc/filegate.yaml)")
 	rootCmd.PersistentFlags().
 		BoolVarP(&production, "production", "p", true, "Toggle production use")
+	rootCmd.PersistentFlags().
+		StringVarP(&localfs, "localfs", "l", "", "Use this directory instead of S3")
 
 	// Cobra also supports local flags, which will only run
 	// when this action is called directly.

config.yaml

@@ -7,7 +7,7 @@ db:
     type: sqlite
     user: dbuser
 web:
-    address: 127.0.0.1:8080
+    address: 0.0.0.0:8080
     sessionkey: 8Zpj3QEifL036QxxCjM0-MVsuYmt6UPbNYLSC3ljAio1AdctpXMwMaVi5SFbqSu9
     trustedproxies:
         - 127.0.0.1

filegate.service

@@ -0,0 +1,17 @@
+[Service]
+Description=Filegate
+Documentation=https://git.jmbit.de/filegate/filegate
+Wants=network-online.target
+After=minio.service
+
+[Service]
+WorkingDirectory=/var/lib/filegate
+User=filegate
+Group=filegate
+ProtectProc=true
+ExecStart=/usr/local/bin/filegate --config /etc/filegate/config.yaml
+Restart=always
+
+
+[Install]
+Wantedby=multi-user.target

files/downloadFile.go

@@ -12,6 +12,7 @@ import (
 	"github.com/spf13/viper"
 
 	"git.jmbit.de/filegate/filegate/db"
+	"git.jmbit.de/filegate/filegate/files/localfiles"
 )
 
 // DownloadFile Downloads file from a given URL, stores it in a Minio object. This function also calls the Static
@@ -44,17 +45,27 @@ func DownloadFile(rawURL string, blob string, id uint) (uint, error) {
 		contentType = "application/octet-stream"
 	}
 	db.UpdateOriginalName(id, fileName)
-	objectInfo, err := MinioClient.PutObject(ctx, viper.GetString("minio.bucket"), blob, response.Body, response.ContentLength,
-		minio.PutObjectOptions{ContentType: contentType})
-	if err != nil {
-		log.Printf("Could not create File object: %v \n", err)
-		return 0, err
-	}
-	log.Printf("Successfully uploaded %s of size %d\n", blob, objectInfo.Size)
-	if err != nil {
-		log.Println(err)
-		return 0, err
-	}
+  if viper.GetString("localfs") == "" {
+	  objectInfo, err := MinioClient.PutObject(ctx, viper.GetString("minio.bucket"), blob, response.Body, response.ContentLength,
+	  	minio.PutObjectOptions{ContentType: contentType})
+	  if err != nil {
+	  	log.Printf("Could not create File object: %v \n", err)
+	  	return 0, err
+	  }
+	  log.Printf("Successfully uploaded %s of size %d\n", blob, objectInfo.Size)
+	  if err != nil {
+	  	log.Println(err)
+	  	return 0, err
+	  }
+  } else {
+    err := localfiles.PutObject(viper.GetString("localfs"), blob, response.Body, response.ContentLength, contentType)
+    if err != nil {
+	  	log.Printf("Could not create File object: %v \n", err)
+	  	return 0, err
+    }
+  }
+
+
   // go RunStaticAnalysis(id)
 	return 0, nil
 }

files/filemanager.go

@@ -10,6 +10,7 @@ import (
   "github.com/google/uuid"
 
 	"git.jmbit.de/filegate/filegate/db"
+	"git.jmbit.de/filegate/filegate/files/localfiles"
 )
 
 func UploadFile(file *multipart.FileHeader, name string, url string, comment string) (uint, error) {
@@ -21,19 +22,34 @@ func UploadFile(file *multipart.FileHeader, name string, url string, comment str
 		return 0, err
 	}
   fileSize := file.Size
+  contentType := file.Header.Get("ContentType")
   fileID, err := db.CreateFile(name, url, comment, blob)
 	if err != nil {
 		log.Printf("Could not create File object: %v \n", err)
 		return fileID, err
 	}
 	db.UpdateOriginalName(fileID, file.Filename)
-	objectInfo, err := MinioClient.PutObject(ctx, viper.GetString("minio.bucket"), blob, fileReader, fileSize,
-		minio.PutObjectOptions{ContentType: "application/octet-stream"})
-	if err != nil {
-		log.Printf("Could not create File object: %v \n", err)
-		return fileID, err
-	}
-	log.Printf("Successfully uploaded %s of size %d\n", blob, objectInfo.Size)
+  if viper.GetString("localfs") == "" {
+	  objectInfo, err := MinioClient.PutObject(ctx, viper.GetString("minio.bucket"), blob, fileReader, file.Size,
+	  	minio.PutObjectOptions{ContentType: contentType})
+	  if err != nil {
+	  	log.Printf("Could not create File object: %v \n", err)
+	  	return 0, err
+	  }
+	  log.Printf("Successfully uploaded %s of size %d\n", blob, objectInfo.Size)
+	  if err != nil {
+	  	log.Println(err)
+	  	return 0, err
+	  }
+	  log.Printf("Successfully uploaded %s of size %d\n", blob, objectInfo.Size)
+  } else {
+    err := localfiles.PutObject(viper.GetString("localfs"), blob, fileReader, file.Size, contentType)
+    if err != nil {
+	  	log.Printf("Could not create File object: %v \n", err)
+	  	return 0, err
+    }
+  }
+
 	// go service.RunStaticAnalysis(fileID)
 	return fileID, nil
 }
@@ -45,3 +61,4 @@ func GetFile(uuid string) (*minio.Object, error) {
 
 	return object, err
 }
+

files/localfiles/localfiles.go

@@ -0,0 +1,16 @@
+package localfiles
+
+import (
+	"io"
+)
+
+func PutObject(
+	path string,
+	blob string,
+	reader io.Reader,
+	length int64,
+	mimeType string,
+) error {
+
+	return nil
+}

files/minio.go

@@ -11,9 +11,9 @@ import (
 	"github.com/spf13/viper"
 )
 
-var MinioClient = minioConnect()
+var MinioClient *minio.Client
 
-func minioConnect() *minio.Client {
+func MinioConnect() {
 	endpoint := fmt.Sprintf("%s:%d", viper.GetString("minio.hostname"), viper.GetInt("minio.port"))
 	log.Printf("Minio Endpoint: %s", endpoint)
 	accessKeyID := viper.GetString("minio.accessKeyID")
@@ -28,13 +28,12 @@ func minioConnect() *minio.Client {
 		log.Fatalf("Could not connect to Minio storage: %v", err)
 	}
 
-	return minioClient
-
+	MinioClient = minioClient
 }
 
-func minioSetup() {
+func MinioSetup() {
 	ctx := context.Background()
-	client := minioConnect()
+	client := MinioClient
 	bucketName := viper.GetString("minio.bucket")
 	err := client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: ""})
 	if err != nil {

vagrant-minio-defaults

@@ -0,0 +1,19 @@
+#!/bin/sh
+# MINIO_ROOT_USER and MINIO_ROOT_PASSWORD sets the root account for the MinIO server.
+# This user has unrestricted permissions to perform S3 and administrative API operations on any resource in the deployment.
+# Omit to use the default values 'minioadmin:minioadmin'.
+# MinIO recommends setting non-default values as a best practice, regardless of environment
+
+MINIO_ROOT_USER=minioadmin
+MINIO_ROOT_PASSWORD=Brethren2-Nativity-Mashing-Catalog-Freeway
+
+# MINIO_VOLUMES sets the storage volume or path to use for the MinIO server.
+
+MINIO_VOLUMES="/mnt/data"
+
+# MINIO_SERVER_URL sets the hostname of the local machine for use with the MinIO Server
+# MinIO assumes your network control plane can correctly resolve this hostname to the local machine
+
+# Uncomment the following line and replace the value with the correct hostname for the local machine and port for the MinIO server (9000 by default).
+
+MINIO_SERVER_URL="http://filegate:9000"