From b81cd6128afad19c5fc9b3241f93add02d513159 Mon Sep 17 00:00:00 2001 From: kolaente Date: Fri, 18 Oct 2019 17:30:25 +0200 Subject: [PATCH] Fixed error when setting max file size on 32-Bit systems --- config.yml.sample | 5 +- docs/content/doc/setup/config.md | 5 +- go.mod | 1 + go.sum | 2 + pkg/config/config.go | 2 +- pkg/files/error.go | 2 +- pkg/files/files.go | 13 +- pkg/files/files_test.go | 2 +- pkg/models/error.go | 2 +- pkg/models/task_attachment.go | 2 +- pkg/models/task_attachment_test.go | 2 +- pkg/routes/api/v1/info.go | 4 +- pkg/routes/api/v1/task_attachment.go | 2 +- vendor/github.com/c2h5oh/datasize/.gitignore | 24 ++ vendor/github.com/c2h5oh/datasize/.travis.yml | 11 + vendor/github.com/c2h5oh/datasize/LICENSE | 21 ++ vendor/github.com/c2h5oh/datasize/README.md | 66 ++++++ vendor/github.com/c2h5oh/datasize/datasize.go | 217 ++++++++++++++++++ vendor/modules.txt | 2 + 19 files changed, 369 insertions(+), 16 deletions(-) create mode 100644 vendor/github.com/c2h5oh/datasize/.gitignore create mode 100644 vendor/github.com/c2h5oh/datasize/.travis.yml create mode 100644 vendor/github.com/c2h5oh/datasize/LICENSE create mode 100644 vendor/github.com/c2h5oh/datasize/README.md create mode 100644 vendor/github.com/c2h5oh/datasize/datasize.go diff --git a/config.yml.sample b/config.yml.sample index 78c2211e..860942ca 100644 --- a/config.yml.sample +++ b/config.yml.sample @@ -110,5 +110,6 @@ ratelimit: files: # The path where files are stored basepath: ./files # relative to the binary - # The maximum size of a file, in bytes. - maxsize: 21474836480 # 20 MB + # The maximum size of a file, as a human-readable string. + # Warning: The max size is limited 2^64-1 bytes due to the underlying datatype + maxsize: 20MB diff --git a/docs/content/doc/setup/config.md b/docs/content/doc/setup/config.md index 615cf103..4d840569 100644 --- a/docs/content/doc/setup/config.md +++ b/docs/content/doc/setup/config.md @@ -153,6 +153,7 @@ ratelimit: files: # The path where files are stored basepath: ./files # relative to the binary - # The maximum size of a file, in bytes. - maxsize: 21474836480 # 20 MB + # The maximum size of a file, as a human-readable string. + # Warning: The max size is limited 2^64-1 bytes due to the underlying datatype + maxsize: 20MB {{< /highlight >}} diff --git a/go.mod b/go.mod index ff3e1d3c..cb106c23 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a github.com/beevik/etree v1.1.0 // indirect + github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae github.com/client9/misspell v0.3.4 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/creack/pty v1.1.9 // indirect diff --git a/go.sum b/go.sum index 5fcd27ba..a38c0071 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae h1:2Zmk+8cNvAGuY8AyvZuWpUdpQUAXwfom4ReVMe/CTIo= +github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= diff --git a/pkg/config/config.go b/pkg/config/config.go index 5ddad244..1dad0167 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -197,7 +197,7 @@ func InitDefaultConfig() { RateLimitStore.setDefault("memory") // Files FilesBasePath.setDefault("files") - FilesMaxSize.setDefault(21474836480) // 20 MB + FilesMaxSize.setDefault("20MB") } // InitConfig initializes the config, sets defaults etc. diff --git a/pkg/files/error.go b/pkg/files/error.go index 980987b6..5f987594 100644 --- a/pkg/files/error.go +++ b/pkg/files/error.go @@ -37,7 +37,7 @@ func IsErrFileDoesNotExist(err error) bool { // ErrFileIsTooLarge defines an error where a file is larger than the configured limit type ErrFileIsTooLarge struct { - Size int64 + Size uint64 } // Error is the error implementation of ErrFileIsTooLarge diff --git a/pkg/files/files.go b/pkg/files/files.go index 8f4a63e9..7290ad25 100644 --- a/pkg/files/files.go +++ b/pkg/files/files.go @@ -19,6 +19,7 @@ package files import ( "code.vikunja.io/api/pkg/config" "code.vikunja.io/web" + "github.com/c2h5oh/datasize" "github.com/spf13/afero" "io" "strconv" @@ -30,7 +31,7 @@ type File struct { ID int64 `xorm:"int(11) autoincr not null unique pk" json:"id"` Name string `xorm:"text not null" json:"name"` Mime string `xorm:"text null" json:"mime"` - Size int64 `xorm:"int(11) not null" json:"size"` + Size uint64 `xorm:"int(11) not null" json:"size"` Created time.Time `xorm:"-" json:"created"` @@ -66,9 +67,15 @@ func (f *File) LoadFileMetaByID() (err error) { } // Create creates a new file from an FileHeader -func Create(f io.ReadCloser, realname string, realsize int64, a web.Auth) (file *File, err error) { +func Create(f io.ReadCloser, realname string, realsize uint64, a web.Auth) (file *File, err error) { - if realsize > config.FilesMaxSize.GetInt64() { + // Get and parse the configured file size + var maxSize datasize.ByteSize + err = maxSize.UnmarshalText([]byte(config.FilesMaxSize.GetString())) + if err != nil { + return nil, err + } + if realsize > maxSize.Bytes() { return nil, ErrFileIsTooLarge{Size: realsize} } diff --git a/pkg/files/files_test.go b/pkg/files/files_test.go index 1b03b605..b1f56cef 100644 --- a/pkg/files/files_test.go +++ b/pkg/files/files_test.go @@ -66,7 +66,7 @@ func TestCreate(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(1), file.CreatedByID) assert.Equal(t, "testfile", file.Name) - assert.Equal(t, int64(100), file.Size) + assert.Equal(t, uint64(100), file.Size) }) t.Run("Too Large", func(t *testing.T) { diff --git a/pkg/models/error.go b/pkg/models/error.go index 80409943..5839062b 100644 --- a/pkg/models/error.go +++ b/pkg/models/error.go @@ -763,7 +763,7 @@ func (err ErrTaskAttachmentDoesNotExist) HTTPError() web.HTTPError { // ErrTaskAttachmentIsTooLarge represents an error where the user tries to relate a task with itself type ErrTaskAttachmentIsTooLarge struct { - Size int64 + Size uint64 } // IsErrTaskAttachmentIsTooLarge checks if an error is ErrTaskAttachmentIsTooLarge. diff --git a/pkg/models/task_attachment.go b/pkg/models/task_attachment.go index e285e76a..cbd30d36 100644 --- a/pkg/models/task_attachment.go +++ b/pkg/models/task_attachment.go @@ -47,7 +47,7 @@ func (TaskAttachment) TableName() string { // NewAttachment creates a new task attachment // Note: I'm not sure if only accepting an io.ReadCloser and not an afero.File or os.File instead is a good way of doing things. -func (ta *TaskAttachment) NewAttachment(f io.ReadCloser, realname string, realsize int64, a web.Auth) error { +func (ta *TaskAttachment) NewAttachment(f io.ReadCloser, realname string, realsize uint64, a web.Auth) error { // Store the file file, err := files.Create(f, realname, realsize, a) diff --git a/pkg/models/task_attachment_test.go b/pkg/models/task_attachment_test.go index d81b5846..31a3bcee 100644 --- a/pkg/models/task_attachment_test.go +++ b/pkg/models/task_attachment_test.go @@ -111,7 +111,7 @@ func TestTaskAttachment_NewAttachment(t *testing.T) { assert.NoError(t, err) assert.Equal(t, testuser.ID, ta.File.CreatedByID) assert.Equal(t, "testfile", ta.File.Name) - assert.Equal(t, int64(100), ta.File.Size) + assert.Equal(t, uint64(100), ta.File.Size) // Extra test for max size test } diff --git a/pkg/routes/api/v1/info.go b/pkg/routes/api/v1/info.go index ed438091..f9de2cd7 100644 --- a/pkg/routes/api/v1/info.go +++ b/pkg/routes/api/v1/info.go @@ -28,7 +28,7 @@ type vikunjaInfos struct { FrontendURL string `json:"frontend_url"` Motd string `json:"motd"` LinkSharingEnabled bool `json:"link_sharing_enabled"` - MaxFileSize int64 `json:"max_file_size"` + MaxFileSize string `json:"max_file_size"` } // Info is the handler to get infos about this vikunja instance @@ -44,6 +44,6 @@ func Info(c echo.Context) error { FrontendURL: config.ServiceFrontendurl.GetString(), Motd: config.ServiceMotd.GetString(), LinkSharingEnabled: config.ServiceEnableLinkSharing.GetBool(), - MaxFileSize: config.FilesMaxSize.GetInt64(), + MaxFileSize: config.FilesMaxSize.GetString(), }) } diff --git a/pkg/routes/api/v1/task_attachment.go b/pkg/routes/api/v1/task_attachment.go index 81ac7b34..cfc4f5b2 100644 --- a/pkg/routes/api/v1/task_attachment.go +++ b/pkg/routes/api/v1/task_attachment.go @@ -82,7 +82,7 @@ func UploadTaskAttachment(c echo.Context) error { } defer f.Close() - err = ta.NewAttachment(f, file.Filename, file.Size, user) + err = ta.NewAttachment(f, file.Filename, uint64(file.Size), user) if err != nil { r.Errors = append(r.Errors, handler.HandleHTTPError(err, c)) continue diff --git a/vendor/github.com/c2h5oh/datasize/.gitignore b/vendor/github.com/c2h5oh/datasize/.gitignore new file mode 100644 index 00000000..daf913b1 --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/vendor/github.com/c2h5oh/datasize/.travis.yml b/vendor/github.com/c2h5oh/datasize/.travis.yml new file mode 100644 index 00000000..a6ebc037 --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/.travis.yml @@ -0,0 +1,11 @@ +sudo: false + +language: go +go: + - 1.4 + - 1.5 + - 1.6 + - tip + +script: + - go test -v diff --git a/vendor/github.com/c2h5oh/datasize/LICENSE b/vendor/github.com/c2h5oh/datasize/LICENSE new file mode 100644 index 00000000..f2ba916e --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Maciej Lisiewski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/c2h5oh/datasize/README.md b/vendor/github.com/c2h5oh/datasize/README.md new file mode 100644 index 00000000..d21f136b --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/README.md @@ -0,0 +1,66 @@ +# datasize [![Build Status](https://travis-ci.org/c2h5oh/datasize.svg?branch=master)](https://travis-ci.org/c2h5oh/datasize) + +Golang helpers for data sizes + + +### Constants +Just like `time` package provides `time.Second`, `time.Day` constants `datasize` provides: +* `datasize.B` 1 byte +* `datasize.KB` 1 kilobyte +* `datasize.MB` 1 megabyte +* `datasize.GB` 1 gigabyte +* `datasize.TB` 1 terabyte +* `datasize.PB` 1 petabyte +* `datasize.EB` 1 exabyte + +### Helpers +Just like `time` package provides `duration.Nanoseconds() uint64 `, `duration.Hours() float64` helpers `datasize` has +* `ByteSize.Bytes() uint64` +* `ByteSize.Kilobytes() float4` +* `ByteSize.Megabytes() float64` +* `ByteSize.Gigabytes() float64` +* `ByteSize.Terabytes() float64` +* `ByteSize.Petebytes() float64` +* `ByteSize.Exabytes() float64` + +Warning: see limitations at the end of this document about a possible precission loss + +### Parsing strings +`datasize.ByteSize` implements `TextUnmarshaler` interface and will automatically parse human readable strings into correct values where it is used: +* `"10 MB"` -> `10* datasize.MB` +* `"10240 g"` -> `10 * datasize.TB` +* `"2000"` -> `2000 * datasize.B` +* `"1tB"` -> `datasize.TB` +* `"5 peta"` -> `5 * datasize.PB` +* `"28 kilobytes"` -> `28 * datasize.KB` +* `"1 gigabyte"` -> `1 * datasize.GB` + +You can also do it manually: +```go +var v datasize.ByteSize +err := v.UnmarshalText([]byte("100 mb")) +``` + +### Printing +`Bytesize.String()` uses largest unit allowing an integer value: + * `(102400 * datasize.MB).String()` -> `"100GB"` + * `(datasize.MB + datasize.KB).String()` -> `"1025KB"` + +Use `%d` format string to get value in bytes without a unit + +### JSON and other encoding +Both `TextMarshaler` and `TextUnmarshaler` interfaces are implemented - JSON will just work. Other encoders will work provided they use those interfaces. + +### Human readable +`ByteSize.HumanReadable()` or `ByteSize.HR()` returns a string with 1-3 digits, followed by 1 decimal place, a space and unit big enough to get 1-3 digits + + * `(102400 * datasize.MB).String()` -> `"100.0 GB"` + * `(datasize.MB + 512 * datasize.KB).String()` -> `"1.5 MB"` + +### Limitations +* The underlying data type for `data.ByteSize` is `uint64`, so values outside of 0 to 2^64-1 range will overflow +* size helper functions (like `ByteSize.Kilobytes()`) return `float64`, which can't represent all possible values of `uint64` accurately: + * if the returned value is supposed to have no fraction (ie `(10 * datasize.MB).Kilobytes()`) accuracy loss happens when value is more than 2^53 larger than unit: `.Kilobytes()` over 8 petabytes, `.Megabytes()` over 8 exabytes + * if the returned value is supposed to have a fraction (ie `(datasize.PB + datasize.B).Megabytes()`) in addition to the above note accuracy loss may occur in fractional part too - larger integer part leaves fewer bytes to store fractional part, the smaller the remainder vs unit the move bytes are required to store the fractional part +* Parsing a string with `Mb`, `Tb`, etc units will return a syntax error, because capital followed by lower case is commonly used for bits, not bytes +* Parsing a string with value exceeding 2^64-1 bytes will return 2^64-1 and an out of range error diff --git a/vendor/github.com/c2h5oh/datasize/datasize.go b/vendor/github.com/c2h5oh/datasize/datasize.go new file mode 100644 index 00000000..67547881 --- /dev/null +++ b/vendor/github.com/c2h5oh/datasize/datasize.go @@ -0,0 +1,217 @@ +package datasize + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +type ByteSize uint64 + +const ( + B ByteSize = 1 + KB = B << 10 + MB = KB << 10 + GB = MB << 10 + TB = GB << 10 + PB = TB << 10 + EB = PB << 10 + + fnUnmarshalText string = "UnmarshalText" + maxUint64 uint64 = (1 << 64) - 1 + cutoff uint64 = maxUint64 / 10 +) + +var ErrBits = errors.New("unit with capital unit prefix and lower case unit (b) - bits, not bytes ") + +func (b ByteSize) Bytes() uint64 { + return uint64(b) +} + +func (b ByteSize) KBytes() float64 { + v := b / KB + r := b % KB + return float64(v) + float64(r)/float64(KB) +} + +func (b ByteSize) MBytes() float64 { + v := b / MB + r := b % MB + return float64(v) + float64(r)/float64(MB) +} + +func (b ByteSize) GBytes() float64 { + v := b / GB + r := b % GB + return float64(v) + float64(r)/float64(GB) +} + +func (b ByteSize) TBytes() float64 { + v := b / TB + r := b % TB + return float64(v) + float64(r)/float64(TB) +} + +func (b ByteSize) PBytes() float64 { + v := b / PB + r := b % PB + return float64(v) + float64(r)/float64(PB) +} + +func (b ByteSize) EBytes() float64 { + v := b / EB + r := b % EB + return float64(v) + float64(r)/float64(EB) +} + +func (b ByteSize) String() string { + switch { + case b == 0: + return fmt.Sprint("0B") + case b%EB == 0: + return fmt.Sprintf("%dEB", b/EB) + case b%PB == 0: + return fmt.Sprintf("%dPB", b/PB) + case b%TB == 0: + return fmt.Sprintf("%dTB", b/TB) + case b%GB == 0: + return fmt.Sprintf("%dGB", b/GB) + case b%MB == 0: + return fmt.Sprintf("%dMB", b/MB) + case b%KB == 0: + return fmt.Sprintf("%dKB", b/KB) + default: + return fmt.Sprintf("%dB", b) + } +} + +func (b ByteSize) HR() string { + return b.HumanReadable() +} + +func (b ByteSize) HumanReadable() string { + switch { + case b > EB: + return fmt.Sprintf("%.1f EB", b.EBytes()) + case b > PB: + return fmt.Sprintf("%.1f PB", b.PBytes()) + case b > TB: + return fmt.Sprintf("%.1f TB", b.TBytes()) + case b > GB: + return fmt.Sprintf("%.1f GB", b.GBytes()) + case b > MB: + return fmt.Sprintf("%.1f MB", b.MBytes()) + case b > KB: + return fmt.Sprintf("%.1f KB", b.KBytes()) + default: + return fmt.Sprintf("%d B", b) + } +} + +func (b ByteSize) MarshalText() ([]byte, error) { + return []byte(b.String()), nil +} + +func (b *ByteSize) UnmarshalText(t []byte) error { + var val uint64 + var unit string + + // copy for error message + t0 := t + + var c byte + var i int + +ParseLoop: + for i < len(t) { + c = t[i] + switch { + case '0' <= c && c <= '9': + if val > cutoff { + goto Overflow + } + + c = c - '0' + val *= 10 + + if val > val+uint64(c) { + // val+v overflows + goto Overflow + } + val += uint64(c) + i++ + + default: + if i == 0 { + goto SyntaxError + } + break ParseLoop + } + } + + unit = strings.TrimSpace(string(t[i:])) + switch unit { + case "Kb", "Mb", "Gb", "Tb", "Pb", "Eb": + goto BitsError + } + unit = strings.ToLower(unit) + switch unit { + case "", "b", "byte": + // do nothing - already in bytes + + case "k", "kb", "kilo", "kilobyte", "kilobytes": + if val > maxUint64/uint64(KB) { + goto Overflow + } + val *= uint64(KB) + + case "m", "mb", "mega", "megabyte", "megabytes": + if val > maxUint64/uint64(MB) { + goto Overflow + } + val *= uint64(MB) + + case "g", "gb", "giga", "gigabyte", "gigabytes": + if val > maxUint64/uint64(GB) { + goto Overflow + } + val *= uint64(GB) + + case "t", "tb", "tera", "terabyte", "terabytes": + if val > maxUint64/uint64(TB) { + goto Overflow + } + val *= uint64(TB) + + case "p", "pb", "peta", "petabyte", "petabytes": + if val > maxUint64/uint64(PB) { + goto Overflow + } + val *= uint64(PB) + + case "E", "EB", "e", "eb", "eB": + if val > maxUint64/uint64(EB) { + goto Overflow + } + val *= uint64(EB) + + default: + goto SyntaxError + } + + *b = ByteSize(val) + return nil + +Overflow: + *b = ByteSize(maxUint64) + return &strconv.NumError{fnUnmarshalText, string(t0), strconv.ErrRange} + +SyntaxError: + *b = 0 + return &strconv.NumError{fnUnmarshalText, string(t0), strconv.ErrSyntax} + +BitsError: + *b = 0 + return &strconv.NumError{fnUnmarshalText, string(t0), ErrBits} +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 23b59675..6ec02530 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -18,6 +18,8 @@ github.com/asaskevich/govalidator github.com/beevik/etree # github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 github.com/beorn7/perks/quantile +# github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae +github.com/c2h5oh/datasize # github.com/client9/misspell v0.3.4 github.com/client9/misspell/cmd/misspell github.com/client9/misspell