Fixed error when setting max file size on 32-Bit systems

config.yml.sample

@@ -110,5 +110,6 @@ ratelimit:
 files:
   # The path where files are stored
   basepath: ./files # relative to the binary
-  # The maximum size of a file, in bytes.
+  # The maximum size of a file, as a human-readable string.
-  maxsize: 21474836480 # 20 MB
+  # Warning: The max size is limited 2^64-1 bytes due to the underlying datatype
+  maxsize: 20MB

docs/content/doc/setup/config.md

@@ -153,6 +153,7 @@ ratelimit:
 files:
   # The path where files are stored
   basepath: ./files # relative to the binary
-  # The maximum size of a file, in bytes.
+  # The maximum size of a file, as a human-readable string.
-  maxsize: 21474836480 # 20 MB
+  # Warning: The max size is limited 2^64-1 bytes due to the underlying datatype
+  maxsize: 20MB
 {{< /highlight >}}

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
 	github.com/beevik/etree v1.1.0 // indirect
+	github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae
 	github.com/client9/misspell v0.3.4
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/creack/pty v1.1.9 // indirect

go.sum

@@ -29,6 +29,8 @@ github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae h1:2Zmk+8cNvAGuY8AyvZuWpUdpQUAXwfom4ReVMe/CTIo=
+github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
 github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=

pkg/config/config.go

@@ -197,7 +197,7 @@ func InitDefaultConfig() {
 	RateLimitStore.setDefault("memory")
 	// Files
 	FilesBasePath.setDefault("files")
-	FilesMaxSize.setDefault(21474836480) // 20 MB
+	FilesMaxSize.setDefault("20MB")
 }
 
 // InitConfig initializes the config, sets defaults etc.

pkg/files/error.go

@@ -37,7 +37,7 @@ func IsErrFileDoesNotExist(err error) bool {
 
 // ErrFileIsTooLarge defines an error where a file is larger than the configured limit
 type ErrFileIsTooLarge struct {
-	Size int64
+	Size uint64
 }
 
 // Error is the error implementation of ErrFileIsTooLarge

pkg/files/files.go

@@ -19,6 +19,7 @@ package files
 import (
 	"code.vikunja.io/api/pkg/config"
 	"code.vikunja.io/web"
+	"github.com/c2h5oh/datasize"
 	"github.com/spf13/afero"
 	"io"
 	"strconv"
@@ -30,7 +31,7 @@ type File struct {
 	ID   int64  `xorm:"int(11) autoincr not null unique pk" json:"id"`
 	Name string `xorm:"text not null" json:"name"`
 	Mime string `xorm:"text null" json:"mime"`
-	Size int64  `xorm:"int(11) not null" json:"size"`
+	Size uint64 `xorm:"int(11) not null" json:"size"`
 
 	Created time.Time `xorm:"-" json:"created"`
 
@@ -66,9 +67,15 @@ func (f *File) LoadFileMetaByID() (err error) {
 }
 
 // Create creates a new file from an FileHeader
-func Create(f io.ReadCloser, realname string, realsize int64, a web.Auth) (file *File, err error) {
+func Create(f io.ReadCloser, realname string, realsize uint64, a web.Auth) (file *File, err error) {
 
-	if realsize > config.FilesMaxSize.GetInt64() {
+	// Get and parse the configured file size
+	var maxSize datasize.ByteSize
+	err = maxSize.UnmarshalText([]byte(config.FilesMaxSize.GetString()))
+	if err != nil {
+		return nil, err
+	}
+	if realsize > maxSize.Bytes() {
 		return nil, ErrFileIsTooLarge{Size: realsize}
 	}
 

pkg/files/files_test.go

@@ -66,7 +66,7 @@ func TestCreate(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, int64(1), file.CreatedByID)
 		assert.Equal(t, "testfile", file.Name)
-		assert.Equal(t, int64(100), file.Size)
+		assert.Equal(t, uint64(100), file.Size)
 
 	})
 	t.Run("Too Large", func(t *testing.T) {

pkg/models/error.go

@@ -763,7 +763,7 @@ func (err ErrTaskAttachmentDoesNotExist) HTTPError() web.HTTPError {
 
 // ErrTaskAttachmentIsTooLarge represents an error where the user tries to relate a task with itself
 type ErrTaskAttachmentIsTooLarge struct {
-	Size int64
+	Size uint64
 }
 
 // IsErrTaskAttachmentIsTooLarge checks if an error is ErrTaskAttachmentIsTooLarge.

pkg/models/task_attachment.go

@@ -47,7 +47,7 @@ func (TaskAttachment) TableName() string {
 
 // NewAttachment creates a new task attachment
 // Note: I'm not sure if only accepting an io.ReadCloser and not an afero.File or os.File instead is a good way of doing things.
-func (ta *TaskAttachment) NewAttachment(f io.ReadCloser, realname string, realsize int64, a web.Auth) error {
+func (ta *TaskAttachment) NewAttachment(f io.ReadCloser, realname string, realsize uint64, a web.Auth) error {
 
 	// Store the file
 	file, err := files.Create(f, realname, realsize, a)

pkg/models/task_attachment_test.go

@@ -111,7 +111,7 @@ func TestTaskAttachment_NewAttachment(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, testuser.ID, ta.File.CreatedByID)
 	assert.Equal(t, "testfile", ta.File.Name)
-	assert.Equal(t, int64(100), ta.File.Size)
+	assert.Equal(t, uint64(100), ta.File.Size)
 
 	// Extra test for max size test
 }

pkg/routes/api/v1/info.go

@@ -28,7 +28,7 @@ type vikunjaInfos struct {
 	FrontendURL        string `json:"frontend_url"`
 	Motd               string `json:"motd"`
 	LinkSharingEnabled bool   `json:"link_sharing_enabled"`
-	MaxFileSize        int64  `json:"max_file_size"`
+	MaxFileSize        string `json:"max_file_size"`
 }
 
 // Info is the handler to get infos about this vikunja instance
@@ -44,6 +44,6 @@ func Info(c echo.Context) error {
 		FrontendURL:        config.ServiceFrontendurl.GetString(),
 		Motd:               config.ServiceMotd.GetString(),
 		LinkSharingEnabled: config.ServiceEnableLinkSharing.GetBool(),
-		MaxFileSize:        config.FilesMaxSize.GetInt64(),
+		MaxFileSize:        config.FilesMaxSize.GetString(),
 	})
 }

pkg/routes/api/v1/task_attachment.go

@@ -82,7 +82,7 @@ func UploadTaskAttachment(c echo.Context) error {
 		}
 		defer f.Close()
 
-		err = ta.NewAttachment(f, file.Filename, file.Size, user)
+		err = ta.NewAttachment(f, file.Filename, uint64(file.Size), user)
 		if err != nil {
 			r.Errors = append(r.Errors, handler.HandleHTTPError(err, c))
 			continue

vendor/github.com/c2h5oh/datasize/.gitignore

@@ -0,0 +1,24 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof

vendor/github.com/c2h5oh/datasize/.travis.yml

@@ -0,0 +1,11 @@
+sudo: false
+
+language: go
+go:
+  - 1.4
+  - 1.5
+  - 1.6
+  - tip
+
+script:
+  - go test -v

vendor/github.com/c2h5oh/datasize/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Maciej Lisiewski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/c2h5oh/datasize/README.md

@@ -0,0 +1,66 @@
+# datasize [![Build Status](https://travis-ci.org/c2h5oh/datasize.svg?branch=master)](https://travis-ci.org/c2h5oh/datasize)
+
+Golang helpers for data sizes
+
+
+### Constants
+Just like `time` package provides `time.Second`, `time.Day` constants `datasize` provides:
+* `datasize.B` 1 byte
+* `datasize.KB` 1 kilobyte
+* `datasize.MB` 1 megabyte
+* `datasize.GB` 1 gigabyte
+* `datasize.TB` 1 terabyte
+* `datasize.PB` 1 petabyte
+* `datasize.EB` 1 exabyte
+
+### Helpers
+Just like `time` package provides `duration.Nanoseconds() uint64 `, `duration.Hours() float64` helpers `datasize` has
+* `ByteSize.Bytes() uint64`
+* `ByteSize.Kilobytes() float4`
+* `ByteSize.Megabytes() float64`
+* `ByteSize.Gigabytes() float64`
+* `ByteSize.Terabytes() float64`
+* `ByteSize.Petebytes() float64`
+* `ByteSize.Exabytes() float64`
+
+Warning: see limitations at the end of this document about a possible precission loss
+
+### Parsing strings
+`datasize.ByteSize` implements `TextUnmarshaler` interface and will automatically parse human readable strings into correct values where it is used:
+* `"10 MB"` -> `10* datasize.MB`
+* `"10240 g"` -> `10 * datasize.TB`
+* `"2000"` -> `2000 * datasize.B`
+* `"1tB"` -> `datasize.TB`
+* `"5 peta"` -> `5 * datasize.PB`
+* `"28 kilobytes"` -> `28 * datasize.KB`
+* `"1 gigabyte"` -> `1 * datasize.GB`
+
+You can also do it manually:
+```go
+var v datasize.ByteSize
+err := v.UnmarshalText([]byte("100 mb"))
+```
+
+### Printing
+`Bytesize.String()` uses largest unit allowing an integer value:
+    * `(102400 * datasize.MB).String()` -> `"100GB"`
+    * `(datasize.MB + datasize.KB).String()` -> `"1025KB"`
+
+Use `%d` format string to get value in bytes without a unit
+
+### JSON and other encoding
+Both `TextMarshaler` and `TextUnmarshaler` interfaces are implemented - JSON will just work. Other encoders will work provided they use those interfaces.
+
+### Human readable
+`ByteSize.HumanReadable()` or `ByteSize.HR()` returns a string with 1-3 digits, followed by 1 decimal place, a space and unit big enough to get 1-3 digits
+
+    * `(102400 * datasize.MB).String()` -> `"100.0 GB"`
+    * `(datasize.MB + 512 * datasize.KB).String()` -> `"1.5 MB"`
+
+### Limitations
+* The underlying data type for `data.ByteSize` is `uint64`, so values outside of 0 to 2^64-1 range will overflow
+* size helper functions (like `ByteSize.Kilobytes()`) return `float64`, which can't represent all possible values of `uint64` accurately:
+  * if the returned value is supposed to have no fraction (ie `(10 * datasize.MB).Kilobytes()`) accuracy loss happens when value is more than 2^53 larger than unit: `.Kilobytes()` over 8 petabytes, `.Megabytes()` over 8 exabytes
+  * if the returned value is supposed to have a fraction (ie `(datasize.PB + datasize.B).Megabytes()`) in addition to the above note accuracy loss may occur in fractional part too - larger integer part leaves fewer bytes to store fractional part, the smaller the remainder vs unit the move bytes are required to store the fractional part
+* Parsing a string with `Mb`, `Tb`, etc units will return a syntax error, because capital followed by lower case is commonly used for bits, not bytes
+* Parsing a string with value exceeding 2^64-1 bytes will return 2^64-1 and an out of range error

vendor/github.com/c2h5oh/datasize/datasize.go

@@ -0,0 +1,217 @@
+package datasize
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type ByteSize uint64
+
+const (
+	B  ByteSize = 1
+	KB          = B << 10
+	MB          = KB << 10
+	GB          = MB << 10
+	TB          = GB << 10
+	PB          = TB << 10
+	EB          = PB << 10
+
+	fnUnmarshalText string = "UnmarshalText"
+	maxUint64       uint64 = (1 << 64) - 1
+	cutoff          uint64 = maxUint64 / 10
+)
+
+var ErrBits = errors.New("unit with capital unit prefix and lower case unit (b) - bits, not bytes ")
+
+func (b ByteSize) Bytes() uint64 {
+	return uint64(b)
+}
+
+func (b ByteSize) KBytes() float64 {
+	v := b / KB
+	r := b % KB
+	return float64(v) + float64(r)/float64(KB)
+}
+
+func (b ByteSize) MBytes() float64 {
+	v := b / MB
+	r := b % MB
+	return float64(v) + float64(r)/float64(MB)
+}
+
+func (b ByteSize) GBytes() float64 {
+	v := b / GB
+	r := b % GB
+	return float64(v) + float64(r)/float64(GB)
+}
+
+func (b ByteSize) TBytes() float64 {
+	v := b / TB
+	r := b % TB
+	return float64(v) + float64(r)/float64(TB)
+}
+
+func (b ByteSize) PBytes() float64 {
+	v := b / PB
+	r := b % PB
+	return float64(v) + float64(r)/float64(PB)
+}
+
+func (b ByteSize) EBytes() float64 {
+	v := b / EB
+	r := b % EB
+	return float64(v) + float64(r)/float64(EB)
+}
+
+func (b ByteSize) String() string {
+	switch {
+	case b == 0:
+		return fmt.Sprint("0B")
+	case b%EB == 0:
+		return fmt.Sprintf("%dEB", b/EB)
+	case b%PB == 0:
+		return fmt.Sprintf("%dPB", b/PB)
+	case b%TB == 0:
+		return fmt.Sprintf("%dTB", b/TB)
+	case b%GB == 0:
+		return fmt.Sprintf("%dGB", b/GB)
+	case b%MB == 0:
+		return fmt.Sprintf("%dMB", b/MB)
+	case b%KB == 0:
+		return fmt.Sprintf("%dKB", b/KB)
+	default:
+		return fmt.Sprintf("%dB", b)
+	}
+}
+
+func (b ByteSize) HR() string {
+	return b.HumanReadable()
+}
+
+func (b ByteSize) HumanReadable() string {
+	switch {
+	case b > EB:
+		return fmt.Sprintf("%.1f EB", b.EBytes())
+	case b > PB:
+		return fmt.Sprintf("%.1f PB", b.PBytes())
+	case b > TB:
+		return fmt.Sprintf("%.1f TB", b.TBytes())
+	case b > GB:
+		return fmt.Sprintf("%.1f GB", b.GBytes())
+	case b > MB:
+		return fmt.Sprintf("%.1f MB", b.MBytes())
+	case b > KB:
+		return fmt.Sprintf("%.1f KB", b.KBytes())
+	default:
+		return fmt.Sprintf("%d B", b)
+	}
+}
+
+func (b ByteSize) MarshalText() ([]byte, error) {
+	return []byte(b.String()), nil
+}
+
+func (b *ByteSize) UnmarshalText(t []byte) error {
+	var val uint64
+	var unit string
+
+	// copy for error message
+	t0 := t
+
+	var c byte
+	var i int
+
+ParseLoop:
+	for i < len(t) {
+		c = t[i]
+		switch {
+		case '0' <= c && c <= '9':
+			if val > cutoff {
+				goto Overflow
+			}
+
+			c = c - '0'
+			val *= 10
+
+			if val > val+uint64(c) {
+				// val+v overflows
+				goto Overflow
+			}
+			val += uint64(c)
+			i++
+
+		default:
+			if i == 0 {
+				goto SyntaxError
+			}
+			break ParseLoop
+		}
+	}
+
+	unit = strings.TrimSpace(string(t[i:]))
+	switch unit {
+	case "Kb", "Mb", "Gb", "Tb", "Pb", "Eb":
+		goto BitsError
+	}
+	unit = strings.ToLower(unit)
+	switch unit {
+	case "", "b", "byte":
+		// do nothing - already in bytes
+
+	case "k", "kb", "kilo", "kilobyte", "kilobytes":
+		if val > maxUint64/uint64(KB) {
+			goto Overflow
+		}
+		val *= uint64(KB)
+
+	case "m", "mb", "mega", "megabyte", "megabytes":
+		if val > maxUint64/uint64(MB) {
+			goto Overflow
+		}
+		val *= uint64(MB)
+
+	case "g", "gb", "giga", "gigabyte", "gigabytes":
+		if val > maxUint64/uint64(GB) {
+			goto Overflow
+		}
+		val *= uint64(GB)
+
+	case "t", "tb", "tera", "terabyte", "terabytes":
+		if val > maxUint64/uint64(TB) {
+			goto Overflow
+		}
+		val *= uint64(TB)
+
+	case "p", "pb", "peta", "petabyte", "petabytes":
+		if val > maxUint64/uint64(PB) {
+			goto Overflow
+		}
+		val *= uint64(PB)
+
+	case "E", "EB", "e", "eb", "eB":
+		if val > maxUint64/uint64(EB) {
+			goto Overflow
+		}
+		val *= uint64(EB)
+
+	default:
+		goto SyntaxError
+	}
+
+	*b = ByteSize(val)
+	return nil
+
+Overflow:
+	*b = ByteSize(maxUint64)
+	return &strconv.NumError{fnUnmarshalText, string(t0), strconv.ErrRange}
+
+SyntaxError:
+	*b = 0
+	return &strconv.NumError{fnUnmarshalText, string(t0), strconv.ErrSyntax}
+
+BitsError:
+	*b = 0
+	return &strconv.NumError{fnUnmarshalText, string(t0), ErrBits}
+}

vendor/modules.txt

@@ -18,6 +18,8 @@ github.com/asaskevich/govalidator
 github.com/beevik/etree
 # github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973
 github.com/beorn7/perks/quantile
+# github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae
+github.com/c2h5oh/datasize
 # github.com/client9/misspell v0.3.4
 github.com/client9/misspell/cmd/misspell
 github.com/client9/misspell